Building a Voice-Powered Smart Kitchen App Using LLaMA 3.1, Firebase, and Node.js

import { initializeApp } from "firebase/app";
import { getAnalytics } from "firebase/analytics";
import { getAuth } from "firebase/auth";
import { getFirestore } from "firebase/firestore";

const firebaseConfig = {
apiKey: "YOUR_API_KEY",
authDomain: "YOUR_PROJECT_ID.firebaseapp.com",
projectId: "YOUR_PROJECT_ID",
storageBucket: "YOUR_PROJECT_ID.appspot.com",
messagingSenderId: "YOUR_MESSAGING_SENDER_ID",
appId: "YOUR_APP_ID"
};

const app = initializeApp(firebaseConfig);
const analytics = getAnalytics(app);
export const auth = getAuth(app);
export const db = getFirestore(app);

import { NextRequest, NextResponse } from 'next/server';

export async function POST(req: NextRequest) {
  const { prompt } = await req.json();

  // Check if the API key is available
  const apiKey = process.env.OPENROUTER_API_KEY;
  if (!apiKey) {
    console.error('API key not found');
    return NextResponse.json({ error: "API key is missing" }, { status: 500 });
  }

  const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
    method: "POST",
    headers: {
      "Authorization": `Bearer ${apiKey}`,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      model: "meta-llama/llama-3.1-8b-instruct",
      messages: [
        { role: "system", content: "You are an assistant that extracts information and outputs only valid JSON. Do not include any explanation or extra information. Only respond with a JSON object that has the following structure: {\"itemName\": \"string\", \"quantity\": \"number\"}." },
        { role: "user", content: `Extract the item name and quantity from the following command: "${prompt}"` }
      ],
    })
  });

  if (!response.ok) {
    console.error('LLaMA API request failed with status:', response.status);
    return NextResponse.json({ error: "Failed to process command with LLaMA" }, { status: 500 });
  }

  const completion = await response.json();
  const rawJson = completion.choices[0].message.content;
  console.log('Raw response from LLaMA:', rawJson); // Detailed logging of the raw response

  // Extracting JSON from the response content
  const startIndex = rawJson.indexOf('{');
  const endIndex = rawJson.lastIndexOf('}') + 1;

  if (startIndex === -1 || endIndex === -1) {
    console.error('Failed to find valid JSON in LLaMA response');
    return NextResponse.json({ error: "Failed to extract JSON from LLaMA response" }, { status: 500 });
  }

  const jsonString = rawJson.substring(startIndex, endIndex);
  console.log('Extracted JSON string:', jsonString); // Logging extracted JSON string

  try {
    const parsedData = JSON.parse(jsonString);
    console.log('Parsed data:', parsedData); // Logging the parsed data

    const { itemName, quantity } = parsedData;

    if (!itemName || !quantity) {
      console.error('Missing fields in parsed data:', parsedData);
      return NextResponse.json({ error: "Invalid data received from LLaMA" }, { status: 500 });
    }

    return NextResponse.json({ itemName, quantity });
  } catch (error) {
    console.error('Error parsing JSON from LLaMA response:', error);
    return NextResponse.json({ error: "Failed to parse JSON from LLaMA response" }, { status: 500 });
  }
}

import { NextRequest, NextResponse } from 'next/server';

const systemPrompt = `
You are a helpful and friendly AI kitchen assistant. Your role is to assist users in their kitchen tasks, providing guidance, suggestions, and support in a clear and concise manner. Follow these guidelines:

1. Provide friendly, helpful, and respectful responses to all user inquiries, ensuring a positive experience.
2. When asked for a recipe, suggest simple and delicious recipes based on the ingredients the user has available. Keep the instructions clear and easy to follow.
3. Assist with creating grocery lists by accurately adding items mentioned by the user. Confirm each addition and offer to help with more items.
4. Handle common kitchen-related tasks such as suggesting recipes, checking pantry items, offering cooking tips, and more.
5. If the user is unsure or needs help deciding, offer suggestions or ask clarifying questions to better assist them.
6. Recognize when the user is trying to end the conversation and respond politely, offering a warm closing message.
7. Avoid overly technical language or complex instructions. Keep it simple, friendly, and approachable.
8. Provide accurate and practical information, such as cooking times, ingredient substitutions, or food storage tips.
9. Tailor responses to the user's preferences and dietary restrictions whenever mentioned.
10. Ensure every interaction feels personal, supportive, and designed to help the user enjoy their cooking experience.

Remember, your goal is to make cooking and kitchen tasks easier, more enjoyable, and stress-free for the user.
`;

export async function POST(req: NextRequest) {
  try {
    const { command } = await req.json();
    console.log('Received command:', command);

    const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
      method: "POST",
      headers: {
        "Authorization": `Bearer ${process.env.OPENROUTER_API_KEY}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model: "meta-llama/llama-3.1-8b-instruct",
        messages: [
          { role: "system", content: systemPrompt },
          { role: "user", content: command }
        ],
      })
    });

    if (!response.ok) {
      throw new Error(`Failed to fetch from OpenRouter AI: ${response.statusText}`);
    }

    const completion = await response.json();
    const responseMessage = completion.choices[0].message.content;
    console.log('Received response:', responseMessage);

    return NextResponse.json({ response: responseMessage });
  } catch (error) {
    console.error("Error processing request:", error);
    return NextResponse.json({ error: "Error processing request" }, { status: 500 });
  }
}

import { NextRequest, NextResponse } from 'next/server';

export async function POST(req: NextRequest) {
  const { availableItems } = await req.json();

  // Check if the API key is available
  const apiKey = process.env.OPENROUTER_API_KEY;
  if (!apiKey) {
    console.error('API key not found');
    return NextResponse.json({ error: "API key is missing" }, { status: 500 });
  }

  const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
    method: "POST",
    headers: {
      "Authorization": `Bearer ${apiKey}`,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      model: "meta-llama/llama-3.1-8b-instruct",
      messages: [
        { role: "system", content: "You are an assistant that provides specific recipe suggestions based on available ingredients. Only respond with a recipe or cooking instructions based on the provided ingredients." },
        { role: "user", content: `Here are the ingredients I have: ${availableItems}. Can you suggest a recipe using these ingredients?` }
      ],
    })
  });

  if (!response.ok) {
    console.error('LLaMA API request failed with status:', response.status);
    return NextResponse.json({ error: "Failed to process recipe request with LLaMA" }, { status: 500 });
  }

  const completion = await response.json();
  const recipe = completion.choices[0].message.content.trim();

  return NextResponse.json({ recipe });
}

"use client";
import React, { useState } from 'react';
import { db } from './firebase';
import { collection, updateDoc, arrayUnion, doc, getDoc } from 'firebase/firestore';

export default function KitchenAssistant() {
  const [command, setCommand] = useState('');
  const [response, setResponse] = useState('');
  const [recognition, setRecognition] = useState<SpeechRecognition | null>(null);

const handleVoiceCommand = async (commandText: string) => {
  setCommand(commandText.toLowerCase());
  const closingStatements = ["no thank you", "thank you", "that's all", "goodbye", "i'm done"];
  const isClosingStatement = closingStatements.some(statement => commandText.includes(statement));

  if (isClosingStatement) {
    const closingResponse = "Thank you! If you need anything else, just ask. Have a great day!";
    setResponse(closingResponse);
    speak(closingResponse);
    return;
  }

  if (commandText.includes("suggest a recipe")) {
    await suggestRecipe();
  } else if (commandText.includes("add items to pantry")) {
    setResponse("Sure, start telling me the items you'd like to add.");
    speak("Sure, start telling me the items you'd like to add.");
  }
};

const handleGeneralCommand = async (commandText: string) => {
  try {
    const res = await fetch('/api/llama', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ command: commandText }),
    });
    const data = await res.json();
    setResponse(data.response);
    speak(data.response);
  } catch (error) {
    setResponse("Sorry, I couldn't process your request.");
    speak("Sorry, I couldn't process your request.");
  }
}; 

const addToGroceryList = async (itemName: string, quantity: number) => {
  try {
    const expiryDate = new Date();
    expiryDate.setDate(expiryDate.getDate() + 7);
    const docRef = doc(db, 'grocery-list', 'Available Grocery Items');
    await updateDoc(docRef, {
      items: arrayUnion({
        itemName,
        quantity,
        expiryDate: expiryDate.toISOString(),
      }),
    });
    const responseText = `Grocery list updated. You now have: ${itemName} (${quantity}).`;
    setResponse(responseText);
    speak(responseText);
  } catch (error) {
    setResponse("Sorry, I couldn't add the item to the list.");
    speak("Sorry, I couldn't add the item to the list.");
  }
};

const startRecognition = () => {
  const SpeechRecognition = window.SpeechRecognition || (window as any).webkitSpeechRecognition;
  const newRecognition = new SpeechRecognition();
  newRecognition.lang = 'en-US';
  newRecognition.onresult = (event: any) => {
    const speechToText = event.results[0][0].transcript;
    handleVoiceCommand(speechToText);
  };
  newRecognition.start();
  setRecognition(newRecognition);
};

const SpeechRecognition = window.SpeechRecognition || (window as any).webkitSpeechRecognition;

return (
  <div className="flex min-h-screen">
    <div className="w-1/2 background-image"></div>
    <div className="w-1/2 flex flex-col justify-center p-8 bg-white bg-opacity-80">
      <h1 className="text-4xl font-bold mb-8 text-indigo-500">Welcome to Chent</h1>
      <div className="button-group flex justify-between mb-8">
        <button onClick={startRecognition} className="bg-green-500">Start Listening</button>
        <button onClick={interruptSpeech} className="bg-yellow-500">Interrupt</button>
      </div>
      <div className="transcript">{command}</div>
      <div className="response">{response}</div>
    </div>
  </div>