integrated major speaking functionality

frontend/src/logic/sdk.ts

@@ -1,6 +1,7 @@
 
 import MandarinChinese from '../assets/img/lang/zh.svg'
 import Japanese from '../assets/img/lang/ja.svg'
+import English from '../assets/img/lang/en.svg'
 
 // db.users: Signup table map<username, password>
 // db.user: Current logged-in user
@@ -17,6 +18,7 @@ export interface Lang {
 export const possibleLangs = [
   {name: 'Mandarin Chinese', code: 'zh', icon: MandarinChinese},
   {name: 'Japanese', code: 'ja', icon: Japanese},
+  {name: 'English', code: 'en', icon: English},
 ]
 
 export function signup(username: string, password: string, language: string)
@@ -62,18 +64,25 @@ export function getUsername()
   return db.user
 }
 
- async function post(endpoint: string, body: any) {
+// export function recordAudio(callback: (audio: HTMLAudioElement) => void) {
-  const response = await fetch(`${backendUrl}/${endpoint}`, {
+//   let chunks = [] as any;
-    method: 'POST',
+//   let mediaRecorder = null as any;
-    headers: {
-      'Content-Type': 'application/json'
-    },
-    body: JSON.stringify(body),
-  });
 
-  return response;
+//   navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
+//     mediaRecorder = new MediaRecorder(stream)
+//     mediaRecorder.ondataavailable = (e: any) => {
+//       chunks.push(e.data)
+//     }
 
- }
+//     mediaRecorder.onstop = (e: any) => {
+//       const blob = new Blob(chunks, { type: 'audio/ogg; codecs=opus' })
+//       chunks = []
+//       const audioURL = window.URL.createObjectURL(blob)
+//       const audio = new Audio(audioURL)
+//       callback(audio);
+//     }
+//   })
+// }
 
 export interface CharacterChatCreationRequest 
 {
@@ -84,7 +93,6 @@ export interface CharacterChatCreationRequest
 
 export interface CharacterChatCreationResponse 
 {
-  // this shoudl be of type UUID
   chat_id: string;
 }
 
@@ -97,8 +105,6 @@ export async function startFictionalChat(character: string): Promise<CharacterCh
     user_name: currUser,
     language: language
   };
-
-  console.log('Sending request:', request);
   
   const response = await fetch(`${backendUrl}/character-chat`, {
     method: 'POST',
@@ -113,7 +119,56 @@ export async function startFictionalChat(character: string): Promise<CharacterCh
     throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`);
   }
 
-  return await response.json();
+  const json = await response.json();
+  return json.session_id;
 }
 
-// export interface 
+export async function speechToText(audioBlob: Blob): Promise<string> {
+  const formData = new FormData();
+  formData.append('audio_file', audioBlob, 'audio.wav');
+
+  const response = await fetch(`${backendUrl}/recognize`, {
+      method: 'POST',
+      body: formData,
+  });
+
+  const json = await response.json();
+
+  if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}, message: ${json.message}`);
+  }
+
+  return json.text;
+}
+
+export async function characterChatMessage(sessionId: string, message: string): Promise<{ msg: string, audio_id: string }> {
+
+  const request = {msg : message};
+
+  const response = await fetch(`${backendUrl}/character-chat/${sessionId}`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(request),
+  });
+
+  if (!response.ok) {
+    const errorData = await response.json();
+    throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`);
+  }
+
+  const json = await response.json();
+  return json;
+}
+
+export async function getAudio(audioId: string): Promise<Blob> {
+  const response = await fetch(`${backendUrl}/audio/${audioId}`);
+
+  if (!response.ok) {
+    throw new Error(`HTTP error! status: ${response.status}`);
+  }
+
+  const blob = await response.blob();
+  return blob;
+}

frontend/src/pages/Character.tsx

@@ -2,21 +2,54 @@ import { useLocation, useNavigate } from 'react-router-dom';
 import { Icon } from '@iconify/react';
 import CharacterBadge from '../components/CharacterBadge';
 import { useState } from 'react';
+import { speechToText, characterChatMessage, getAudio } from '../logic/sdk';
 
 export default function Character() {
     const location = useLocation();
     const navigate = useNavigate();
     const { name, image , sessionId } = location.state;
 
-    const [messages, setMessages] = useState([
+    const [messages, setMessages] = useState([{}]);
-        { text: 'Hello!', sender: 'me' },
+
-        { text: 'Hi!', sender: 'other' },
+    let chunks = [] as any;
-        // Add more messages here
+    let mediaRecorder = null as any;
-    ]);
+
+    navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
+        mediaRecorder = new MediaRecorder(stream)
+        mediaRecorder.ondataavailable = (e: any) => {
+            chunks.push(e.data)
+        }
+
+        mediaRecorder.onstop = (e: any) => {
+            const blob = new Blob(chunks, { type: 'audio/wav' });
+            chunks = [];
+        
+            const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' });
+        
+            speechToText(audioFile).then((text) => {
+                setMessages([...messages, { text: text, sender: 'me' }]);
+                characterChatMessage(sessionId, text).then((response) => {
+                    const { msg, audio_id } = response
+                    getAudio(audio_id).then((audioBlob) => {
+                        const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' });
+                        const audioUrl = URL.createObjectURL(audioFile);
+                        const audio = new Audio(audioUrl);
+                        audio.play();
+                    });
+                    setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); 
+                })
+            });
+        }
+    })
 
     function handleRecord() {
-        console.log("Recording...");
+        if (mediaRecorder && mediaRecorder.state === 'inactive') {
-        console.log(sessionId)
+            mediaRecorder.start();
+            console.log("Recording...");
+        } else if (mediaRecorder && mediaRecorder.state === 'recording') {
+            mediaRecorder.stop();
+            console.log("Stopped recording.");
+        }
     }
 
     return (

frontend/src/pages/CharacterSelection.tsx

@@ -15,8 +15,8 @@ export default function CharacterSelection() {
     ];
 
     const handleCharacterClick = (characterName: string, characterImage: string, alias: string) => {
-
         startFictionalChat(alias).then((sessionId) => {
+            console.log(sessionId);
             navigate('/character', { state: { name: characterName, image: characterImage, sessionId: sessionId } });
         })
     }