diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 63ee6b7..4b3e804 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -1,6 +1,7 @@ import MandarinChinese from '../assets/img/lang/zh.svg' import Japanese from '../assets/img/lang/ja.svg' +import English from '../assets/img/lang/en.svg' // db.users: Signup table map // db.user: Current logged-in user @@ -17,6 +18,7 @@ export interface Lang { export const possibleLangs = [ {name: 'Mandarin Chinese', code: 'zh', icon: MandarinChinese}, {name: 'Japanese', code: 'ja', icon: Japanese}, + {name: 'English', code: 'en', icon: English}, ] export function signup(username: string, password: string, language: string) @@ -62,18 +64,25 @@ export function getUsername() return db.user } - async function post(endpoint: string, body: any) { - const response = await fetch(`${backendUrl}/${endpoint}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(body), - }); +// export function recordAudio(callback: (audio: HTMLAudioElement) => void) { +// let chunks = [] as any; +// let mediaRecorder = null as any; - return response; +// navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { +// mediaRecorder = new MediaRecorder(stream) +// mediaRecorder.ondataavailable = (e: any) => { +// chunks.push(e.data) +// } - } +// mediaRecorder.onstop = (e: any) => { +// const blob = new Blob(chunks, { type: 'audio/ogg; codecs=opus' }) +// chunks = [] +// const audioURL = window.URL.createObjectURL(blob) +// const audio = new Audio(audioURL) +// callback(audio); +// } +// }) +// } export interface CharacterChatCreationRequest { @@ -84,7 +93,6 @@ export interface CharacterChatCreationRequest export interface CharacterChatCreationResponse { - // this shoudl be of type UUID chat_id: string; } @@ -97,8 +105,6 @@ export async function startFictionalChat(character: string): Promise { + const formData = new FormData(); + formData.append('audio_file', audioBlob, 'audio.wav'); + + const response = await fetch(`${backendUrl}/recognize`, { + method: 'POST', + body: formData, + }); + + const json = await response.json(); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}, message: ${json.message}`); + } + + return json.text; +} + +export async function characterChatMessage(sessionId: string, message: string): Promise<{ msg: string, audio_id: string }> { + + const request = {msg : message}; + + const response = await fetch(`${backendUrl}/character-chat/${sessionId}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json; +} + +export async function getAudio(audioId: string): Promise { + const response = await fetch(`${backendUrl}/audio/${audioId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const blob = await response.blob(); + return blob; +} \ No newline at end of file diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 23c8fc5..280d361 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -2,21 +2,54 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { Icon } from '@iconify/react'; import CharacterBadge from '../components/CharacterBadge'; import { useState } from 'react'; +import { speechToText, characterChatMessage, getAudio } from '../logic/sdk'; export default function Character() { const location = useLocation(); const navigate = useNavigate(); const { name, image , sessionId } = location.state; - const [messages, setMessages] = useState([ - { text: 'Hello!', sender: 'me' }, - { text: 'Hi!', sender: 'other' }, - // Add more messages here - ]); + const [messages, setMessages] = useState([{}]); + + let chunks = [] as any; + let mediaRecorder = null as any; + + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + mediaRecorder = new MediaRecorder(stream) + mediaRecorder.ondataavailable = (e: any) => { + chunks.push(e.data) + } + + mediaRecorder.onstop = (e: any) => { + const blob = new Blob(chunks, { type: 'audio/wav' }); + chunks = []; + + const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); + + speechToText(audioFile).then((text) => { + setMessages([...messages, { text: text, sender: 'me' }]); + characterChatMessage(sessionId, text).then((response) => { + const { msg, audio_id } = response + getAudio(audio_id).then((audioBlob) => { + const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); + const audioUrl = URL.createObjectURL(audioFile); + const audio = new Audio(audioUrl); + audio.play(); + }); + setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); + }) + }); + } + }) function handleRecord() { - console.log("Recording..."); - console.log(sessionId) + if (mediaRecorder && mediaRecorder.state === 'inactive') { + mediaRecorder.start(); + console.log("Recording..."); + } else if (mediaRecorder && mediaRecorder.state === 'recording') { + mediaRecorder.stop(); + console.log("Stopped recording."); + } } return ( diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index f3b5497..d72e58c 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -15,8 +15,8 @@ export default function CharacterSelection() { ]; const handleCharacterClick = (characterName: string, characterImage: string, alias: string) => { - startFictionalChat(alias).then((sessionId) => { + console.log(sessionId); navigate('/character', { state: { name: characterName, image: characterImage, sessionId: sessionId } }); }) }