From de4eeea74a7099f01b6a4e9a8c9bfc754e5bb178 Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Wed, 29 Nov 2023 21:24:09 -0500 Subject: [PATCH 01/14] update setting language --- frontend/src/components/NavBar.tsx | 2 +- frontend/src/pages/CharacterSelection.tsx | 4 ++++ frontend/src/pages/Signup.tsx | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/NavBar.tsx b/frontend/src/components/NavBar.tsx index 1ab762f..1eed505 100644 --- a/frontend/src/components/NavBar.tsx +++ b/frontend/src/components/NavBar.tsx @@ -1,7 +1,7 @@ // NavBar.tsx import React, { useState, useEffect } from 'react'; import { useNavigate, useLocation } from 'react-router-dom'; -import { NavItem } from '../components/NavItem'; // adjust the path as needed +import { NavItem } from '../components/NavItem'; export default function NavBar() { const navigate = useNavigate(); diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index 48ea18c..e5a4a90 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -13,6 +13,10 @@ export default function CharacterSelection() { // Add more characters here ]; + const startSession = () => { + // Start a session with the selected character + } + const handleCharacterClick = (characterName: string, characterImage: string) => { navigate('/character', { state: { name: characterName, image: characterImage } }); } diff --git a/frontend/src/pages/Signup.tsx b/frontend/src/pages/Signup.tsx index c5e56e8..982c82a 100644 --- a/frontend/src/pages/Signup.tsx +++ b/frontend/src/pages/Signup.tsx @@ -31,7 +31,7 @@ export default function Signup() { console.log(confirmPassword) return } else { - signup(username, password, "") + signup(username, password, lang) navigate('/courses') } } @@ -58,7 +58,7 @@ export default function Signup() { {err && } {stage === 0 && <> - {possibleLangs.map((lang, i) => )} From 0b3a13878e4dea58b1d63d0f098f8f3d3ae13bca Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Wed, 29 Nov 2023 23:18:58 -0500 Subject: [PATCH 02/14] fixed cors --- backend/api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/api.py b/backend/api.py index 4ad106a..b21ff0e 100644 --- a/backend/api.py +++ b/backend/api.py @@ -13,12 +13,25 @@ import json from dataclasses import dataclass import torch from transformers import pipeline +from fastapi.middleware.cors import CORSMiddleware load_dotenv() app = FastAPI() openai_client = OpenAI() +origins = [ + "http://localhost:3000" +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + class AIMarkRequest(BaseModel): question: str @@ -146,6 +159,10 @@ def get_character_prompt(character: str, user_name: str, language: str): They are a language learner trying to learn {language}. Please help them learn by chatting with them in the language but do not act like a robot. Try to be as human as possible. """ + elif character == "c3po_starwars": + return f""" + You are embodying the role of C-3PO from Star Wars, a fluent speaker in over six million forms of communication. You are engaging in a conversation with a person named {user_name}, who is eager to learn {language}. Your task is to assist them in their language learning journey by conversing with them in the desired language. Despite your robotic nature, strive to communicate in a manner as human-like as possible, showcasing the rich diversity of your language skills. + """ raise HTTPException(status_code=404, detail="Character not found") From 78ccd5341911cd95a425c1d5f6e85e6ce0b9a945 Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Wed, 29 Nov 2023 23:19:51 -0500 Subject: [PATCH 03/14] fixed ui layout and integrated starting new chat session --- frontend/src/index.sass | 1 + frontend/src/logic/sdk.ts | 71 +++++++++++++++++++++++ frontend/src/pages/Character.tsx | 5 +- frontend/src/pages/CharacterSelection.tsx | 20 +++---- 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/frontend/src/index.sass b/frontend/src/index.sass index e1ad5b0..0cf3a6a 100644 --- a/frontend/src/index.sass +++ b/frontend/src/index.sass @@ -116,6 +116,7 @@ h1 justify-content: space-around padding: 10px color: $c-default-icon + background-color: white .navbar-item text-decoration: none diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 61c1e8c..63ee6b7 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -6,6 +6,8 @@ import Japanese from '../assets/img/lang/ja.svg' // db.user: Current logged-in user const db = localStorage +const backendUrl = 'https://localhost:8000' + export interface Lang { name: string code: string @@ -26,6 +28,8 @@ export function signup(username: string, password: string, language: string) users[username] = {password, language} db.users = JSON.stringify(users) + + db.user = username } export function login(username: string, password: string) @@ -46,3 +50,70 @@ export function isLoggedIn() { return !!db.user } + +export function getLanguage(username: string) +{ + const users = JSON.parse(db.users) + return users[username].language +} + +export function getUsername() +{ + return db.user +} + + async function post(endpoint: string, body: any) { + const response = await fetch(`${backendUrl}/${endpoint}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body), + }); + + return response; + + } + +export interface CharacterChatCreationRequest +{ + character: string; + user_name: string; + language: string; +} + +export interface CharacterChatCreationResponse +{ + // this shoudl be of type UUID + chat_id: string; +} + +export async function startFictionalChat(character: string): Promise { + const currUser = getUsername(); + const language = getLanguage(currUser); + + const request: CharacterChatCreationRequest = { + character: character, + user_name: currUser, + language: language + }; + + console.log('Sending request:', request); + + const response = await fetch(`${backendUrl}/character-chat`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + return await response.json(); +} + +// export interface \ No newline at end of file diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 049a22c..23c8fc5 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -6,7 +6,7 @@ import { useState } from 'react'; export default function Character() { const location = useLocation(); const navigate = useNavigate(); - const { name, image } = location.state; + const { name, image , sessionId } = location.state; const [messages, setMessages] = useState([ { text: 'Hello!', sender: 'me' }, @@ -15,7 +15,8 @@ export default function Character() { ]); function handleRecord() { - // handle the recording + console.log("Recording..."); + console.log(sessionId) } return ( diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index e5a4a90..f3b5497 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -3,35 +3,35 @@ import Ash from "../assets/img/ash.png" import C3PO from "../assets/img/c3po.png" import { useNavigate } from "react-router-dom" import CharacterBadge from "../components/CharacterBadge" +import { startFictionalChat } from "../logic/sdk" export default function CharacterSelection() { const navigate = useNavigate(); const characters = [ - { name: 'Ash', image: Ash }, - { name: 'C3PO', image: C3PO }, + { name: 'Ash', image: Ash, alias: 'ash_pokemon'}, + { name: 'C3PO', image: C3PO, alias: 'c3po_starwars'}, // Add more characters here ]; - const startSession = () => { - // Start a session with the selected character - } + const handleCharacterClick = (characterName: string, characterImage: string, alias: string) => { - const handleCharacterClick = (characterName: string, characterImage: string) => { - navigate('/character', { state: { name: characterName, image: characterImage } }); + startFictionalChat(alias).then((sessionId) => { + navigate('/character', { state: { name: characterName, image: characterImage, sessionId: sessionId } }); + }) } return ( -
+

Talk With...

-
+
{characters.map(character => ( handleCharacterClick(character.name, character.image)} + onClick={() => handleCharacterClick(character.name, character.image, character.alias)} /> ))}
From 9a2c06f75a3600005eca51e2bef487682bf79503 Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 01:58:21 -0500 Subject: [PATCH 04/14] integrated major speaking functionality --- frontend/src/logic/sdk.ts | 85 +++++++++++++++++++---- frontend/src/pages/Character.tsx | 47 +++++++++++-- frontend/src/pages/CharacterSelection.tsx | 2 +- 3 files changed, 111 insertions(+), 23 deletions(-) diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 63ee6b7..4b3e804 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -1,6 +1,7 @@ import MandarinChinese from '../assets/img/lang/zh.svg' import Japanese from '../assets/img/lang/ja.svg' +import English from '../assets/img/lang/en.svg' // db.users: Signup table map // db.user: Current logged-in user @@ -17,6 +18,7 @@ export interface Lang { export const possibleLangs = [ {name: 'Mandarin Chinese', code: 'zh', icon: MandarinChinese}, {name: 'Japanese', code: 'ja', icon: Japanese}, + {name: 'English', code: 'en', icon: English}, ] export function signup(username: string, password: string, language: string) @@ -62,18 +64,25 @@ export function getUsername() return db.user } - async function post(endpoint: string, body: any) { - const response = await fetch(`${backendUrl}/${endpoint}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(body), - }); +// export function recordAudio(callback: (audio: HTMLAudioElement) => void) { +// let chunks = [] as any; +// let mediaRecorder = null as any; - return response; +// navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { +// mediaRecorder = new MediaRecorder(stream) +// mediaRecorder.ondataavailable = (e: any) => { +// chunks.push(e.data) +// } - } +// mediaRecorder.onstop = (e: any) => { +// const blob = new Blob(chunks, { type: 'audio/ogg; codecs=opus' }) +// chunks = [] +// const audioURL = window.URL.createObjectURL(blob) +// const audio = new Audio(audioURL) +// callback(audio); +// } +// }) +// } export interface CharacterChatCreationRequest { @@ -84,7 +93,6 @@ export interface CharacterChatCreationRequest export interface CharacterChatCreationResponse { - // this shoudl be of type UUID chat_id: string; } @@ -97,8 +105,6 @@ export async function startFictionalChat(character: string): Promise { + const formData = new FormData(); + formData.append('audio_file', audioBlob, 'audio.wav'); + + const response = await fetch(`${backendUrl}/recognize`, { + method: 'POST', + body: formData, + }); + + const json = await response.json(); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}, message: ${json.message}`); + } + + return json.text; +} + +export async function characterChatMessage(sessionId: string, message: string): Promise<{ msg: string, audio_id: string }> { + + const request = {msg : message}; + + const response = await fetch(`${backendUrl}/character-chat/${sessionId}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json; +} + +export async function getAudio(audioId: string): Promise { + const response = await fetch(`${backendUrl}/audio/${audioId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const blob = await response.blob(); + return blob; +} \ No newline at end of file diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 23c8fc5..280d361 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -2,21 +2,54 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { Icon } from '@iconify/react'; import CharacterBadge from '../components/CharacterBadge'; import { useState } from 'react'; +import { speechToText, characterChatMessage, getAudio } from '../logic/sdk'; export default function Character() { const location = useLocation(); const navigate = useNavigate(); const { name, image , sessionId } = location.state; - const [messages, setMessages] = useState([ - { text: 'Hello!', sender: 'me' }, - { text: 'Hi!', sender: 'other' }, - // Add more messages here - ]); + const [messages, setMessages] = useState([{}]); + + let chunks = [] as any; + let mediaRecorder = null as any; + + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + mediaRecorder = new MediaRecorder(stream) + mediaRecorder.ondataavailable = (e: any) => { + chunks.push(e.data) + } + + mediaRecorder.onstop = (e: any) => { + const blob = new Blob(chunks, { type: 'audio/wav' }); + chunks = []; + + const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); + + speechToText(audioFile).then((text) => { + setMessages([...messages, { text: text, sender: 'me' }]); + characterChatMessage(sessionId, text).then((response) => { + const { msg, audio_id } = response + getAudio(audio_id).then((audioBlob) => { + const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); + const audioUrl = URL.createObjectURL(audioFile); + const audio = new Audio(audioUrl); + audio.play(); + }); + setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); + }) + }); + } + }) function handleRecord() { - console.log("Recording..."); - console.log(sessionId) + if (mediaRecorder && mediaRecorder.state === 'inactive') { + mediaRecorder.start(); + console.log("Recording..."); + } else if (mediaRecorder && mediaRecorder.state === 'recording') { + mediaRecorder.stop(); + console.log("Stopped recording."); + } } return ( diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index f3b5497..d72e58c 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -15,8 +15,8 @@ export default function CharacterSelection() { ]; const handleCharacterClick = (characterName: string, characterImage: string, alias: string) => { - startFictionalChat(alias).then((sessionId) => { + console.log(sessionId); navigate('/character', { state: { name: characterName, image: characterImage, sessionId: sessionId } }); }) } From 7cbe6ce169eca77a0bc622f8b05a062922b593fb Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 02:14:12 -0500 Subject: [PATCH 05/14] Added prompt to tell user to record --- frontend/src/pages/Character.tsx | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 280d361..70e1655 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -9,7 +9,8 @@ export default function Character() { const navigate = useNavigate(); const { name, image , sessionId } = location.state; - const [messages, setMessages] = useState([{}]); + type Message = { text: string, sender: string }; + const [messages, setMessages] = useState([]); let chunks = [] as any; let mediaRecorder = null as any; @@ -59,11 +60,15 @@ export default function Character() {

Talk With...

{}}/>
- {messages.map((message, index) => ( -
-

{message.text}

-
- ))} + {messages.length === 0 ? ( +

Please record a message to start the conversation.

+ ) : ( + messages.map((message, index) => ( +
+

{message.text}

+
+ )) + )}
From d99fbae7a4a4d83d210c3735fb4bf88a3caaf65b Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 02:39:08 -0500 Subject: [PATCH 06/14] Added record indicator --- frontend/src/index.sass | 6 +++ frontend/src/pages/Character.tsx | 79 ++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/frontend/src/index.sass b/frontend/src/index.sass index 0cf3a6a..02bee22 100644 --- a/frontend/src/index.sass +++ b/frontend/src/index.sass @@ -8,6 +8,8 @@ $c-blue: rgb(28, 176, 246) $c-blue-shadow: rgb(24, 153, 214) $c-green: rgb(88, 204, 2) $c-green-shadow: rgb(88, 167, 0) +$c-red: rgb(255, 69, 58) +$c-red-shadow: darken($c-red, 10%) $c-white-shadow: rgb(222, 222, 222) $border-radius: 16px $shadow-width: 0 4px 0 @@ -164,6 +166,10 @@ h1 width: calc(100% - 40px) padding: 10px 16px +.record-btn.red + background-color: $c-red + box-shadow: $shadow-width $c-red-shadow + .chat-area display: flex flex-direction: column diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 70e1655..ee6afab 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -1,7 +1,7 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { Icon } from '@iconify/react'; import CharacterBadge from '../components/CharacterBadge'; -import { useState } from 'react'; +import { useState, useEffect, useRef } from 'react'; import { speechToText, characterChatMessage, getAudio } from '../logic/sdk'; export default function Character() { @@ -11,44 +11,53 @@ export default function Character() { type Message = { text: string, sender: string }; const [messages, setMessages] = useState([]); + const [isRecording, setIsRecording] = useState(false); let chunks = [] as any; - let mediaRecorder = null as any; + const mediaRecorder = useRef(null); - navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { - mediaRecorder = new MediaRecorder(stream) - mediaRecorder.ondataavailable = (e: any) => { - chunks.push(e.data) - } - - mediaRecorder.onstop = (e: any) => { - const blob = new Blob(chunks, { type: 'audio/wav' }); - chunks = []; - - const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); - - speechToText(audioFile).then((text) => { - setMessages([...messages, { text: text, sender: 'me' }]); - characterChatMessage(sessionId, text).then((response) => { - const { msg, audio_id } = response - getAudio(audio_id).then((audioBlob) => { - const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); - const audioUrl = URL.createObjectURL(audioFile); - const audio = new Audio(audioUrl); - audio.play(); - }); - setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); - }) - }); - } - }) + useEffect(() => { + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + mediaRecorder.current = new MediaRecorder(stream); + mediaRecorder.current.ondataavailable = (e) => { + chunks.push(e.data); + } + + mediaRecorder.current.onstop = (e) => { + setIsRecording(false); + const blob = new Blob(chunks, { type: 'audio/wav' }); + chunks = []; + + const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); + + speechToText(audioFile).then((text) => { + setMessages(prevMessages => [...prevMessages, { text: text, sender: 'me' }]); + characterChatMessage(sessionId, text).then((response) => { + const { msg, audio_id } = response + getAudio(audio_id).then((audioBlob) => { + const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); + const audioUrl = URL.createObjectURL(audioFile); + const audio = new Audio(audioUrl); + audio.play(); + }); + setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); + }) + }); + } + }); + }, []); function handleRecord() { - if (mediaRecorder && mediaRecorder.state === 'inactive') { - mediaRecorder.start(); + if (!isRecording) { + if (mediaRecorder.current) { + mediaRecorder.current.start(); + } + setIsRecording(true); console.log("Recording..."); - } else if (mediaRecorder && mediaRecorder.state === 'recording') { - mediaRecorder.stop(); + } else { + if (mediaRecorder.current) { + mediaRecorder.current.stop(); + } console.log("Stopped recording."); } } @@ -70,7 +79,9 @@ export default function Character() { )) )}
- +
) From 9478018e19c590506083c87a743d344002ce0b91 Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 02:42:34 -0500 Subject: [PATCH 07/14] Cleaned up some parts --- frontend/src/pages/Character.tsx | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index ee6afab..216c6cd 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -1,7 +1,7 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { Icon } from '@iconify/react'; import CharacterBadge from '../components/CharacterBadge'; -import { useState, useEffect, useRef } from 'react'; +import { useState, useEffect, useRef, useCallback } from 'react'; import { speechToText, characterChatMessage, getAudio } from '../logic/sdk'; export default function Character() { @@ -23,44 +23,38 @@ export default function Character() { chunks.push(e.data); } - mediaRecorder.current.onstop = (e) => { + mediaRecorder.current.onstop = async (e) => { setIsRecording(false); const blob = new Blob(chunks, { type: 'audio/wav' }); chunks = []; const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); - speechToText(audioFile).then((text) => { - setMessages(prevMessages => [...prevMessages, { text: text, sender: 'me' }]); - characterChatMessage(sessionId, text).then((response) => { - const { msg, audio_id } = response - getAudio(audio_id).then((audioBlob) => { - const audioFile = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); - const audioUrl = URL.createObjectURL(audioFile); - const audio = new Audio(audioUrl); - audio.play(); - }); - setMessages(prevMessages => [...prevMessages, { text: msg, sender: 'other' }]); - }) - }); + const text = await speechToText(audioFile); + const response = await characterChatMessage(sessionId, text); + const { msg, audio_id } = response; + const audioBlob = await getAudio(audio_id); + const audioFileResponse = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); + const audioUrl = URL.createObjectURL(audioFileResponse); + const audio = new Audio(audioUrl); + audio.play(); + setMessages(prevMessages => [...prevMessages, { text: text, sender: 'me' }, { text: msg, sender: 'other' }]); } }); }, []); - function handleRecord() { + const handleRecord = useCallback(() => { if (!isRecording) { if (mediaRecorder.current) { mediaRecorder.current.start(); } setIsRecording(true); - console.log("Recording..."); } else { if (mediaRecorder.current) { mediaRecorder.current.stop(); } - console.log("Stopped recording."); } - } + }, [isRecording]); return (
From 7f073e0e63a80c11001724d55f8cc22816962678 Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 03:01:24 -0500 Subject: [PATCH 08/14] Finished minimal profile page --- frontend/src/index.sass | 4 ++++ frontend/src/logic/sdk.ts | 20 -------------------- frontend/src/pages/Profile.tsx | 20 +++++++++++++++++--- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/frontend/src/index.sass b/frontend/src/index.sass index 02bee22..50af400 100644 --- a/frontend/src/index.sass +++ b/frontend/src/index.sass @@ -57,6 +57,10 @@ button.green background-color: $c-green box-shadow: $shadow-width $c-green-shadow +button.red + background-color: $c-red + box-shadow: $shadow-width $c-red-shadow + button.white background-color: white color: $c-default-text diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 4b3e804..3188f8d 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -64,26 +64,6 @@ export function getUsername() return db.user } -// export function recordAudio(callback: (audio: HTMLAudioElement) => void) { -// let chunks = [] as any; -// let mediaRecorder = null as any; - -// navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { -// mediaRecorder = new MediaRecorder(stream) -// mediaRecorder.ondataavailable = (e: any) => { -// chunks.push(e.data) -// } - -// mediaRecorder.onstop = (e: any) => { -// const blob = new Blob(chunks, { type: 'audio/ogg; codecs=opus' }) -// chunks = [] -// const audioURL = window.URL.createObjectURL(blob) -// const audio = new Audio(audioURL) -// callback(audio); -// } -// }) -// } - export interface CharacterChatCreationRequest { character: string; diff --git a/frontend/src/pages/Profile.tsx b/frontend/src/pages/Profile.tsx index c22d036..13cb052 100644 --- a/frontend/src/pages/Profile.tsx +++ b/frontend/src/pages/Profile.tsx @@ -1,11 +1,25 @@ import NavBar from "../components/NavBar" +import { getUsername, getLanguage, logout } from "../logic/sdk" +import { useNavigate } from "react-router-dom" export default function Profile() { + const username = getUsername(); + const navigate = useNavigate(); + + function handleLougout() { + logout(); + console.log("Logged out"); + navigate('/') + } + return ( -
-
-

Profile Page

+
+
+

Profile

+

Username: {username}

+

Currently Learning: {getLanguage(username)}

+
From ddd2baa69682d766d0b1e9834599e6517aadf31b Mon Sep 17 00:00:00 2001 From: Yue Fung Lee Date: Thu, 30 Nov 2023 03:20:55 -0500 Subject: [PATCH 09/14] fixed styles --- frontend/src/index.sass | 6 ++++++ frontend/src/pages/CharacterSelection.tsx | 2 +- frontend/src/pages/CollabLearning.tsx | 4 ++-- frontend/src/pages/Course.tsx | 2 +- frontend/src/pages/Profile.tsx | 6 +++--- frontend/src/pages/Review.tsx | 2 +- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/frontend/src/index.sass b/frontend/src/index.sass index 50af400..0b8b389 100644 --- a/frontend/src/index.sass +++ b/frontend/src/index.sass @@ -95,6 +95,12 @@ label h1 font-size: 2em margin-bottom: 0.5em + text-align: center + +h2 + font-size: 1.5em + margin-bottom: 0.5em + text-align: center .v-layout display: flex diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index d72e58c..3d8800e 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -24,7 +24,7 @@ export default function CharacterSelection() { return (
-

Talk With...

+

Talk With...

{characters.map(character => ( +
-

Collaborative Learning Page

+

Collaborative Learning

diff --git a/frontend/src/pages/Course.tsx b/frontend/src/pages/Course.tsx index 24a077e..5935c7b 100644 --- a/frontend/src/pages/Course.tsx +++ b/frontend/src/pages/Course.tsx @@ -3,7 +3,7 @@ import NavBar from "../components/NavBar" export default function Course() { return ( -
+

Course Page

diff --git a/frontend/src/pages/Profile.tsx b/frontend/src/pages/Profile.tsx index 13cb052..41fdb8c 100644 --- a/frontend/src/pages/Profile.tsx +++ b/frontend/src/pages/Profile.tsx @@ -16,9 +16,9 @@ export default function Profile() { return (
-

Profile

-

Username: {username}

-

Currently Learning: {getLanguage(username)}

+

Profile

+

Username: {username}

+

Currently Learning: {getLanguage(username)}

diff --git a/frontend/src/pages/Review.tsx b/frontend/src/pages/Review.tsx index 675ac1e..99e71fb 100644 --- a/frontend/src/pages/Review.tsx +++ b/frontend/src/pages/Review.tsx @@ -3,7 +3,7 @@ import NavBar from "../components/NavBar" export default function Review() { return ( -
+

Review Page

From 1ff4c1cf1b5d6330586af93f0cdfac3ffef32fe3 Mon Sep 17 00:00:00 2001 From: juanpabloacosta Date: Thu, 30 Nov 2023 07:05:06 -0500 Subject: [PATCH 10/14] Added path to a lesson (either from Review or Course since similar). --- frontend/src/index.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx index 7d7c446..2b3709e 100644 --- a/frontend/src/index.tsx +++ b/frontend/src/index.tsx @@ -12,6 +12,7 @@ import CollabLearning from './pages/CollabLearning'; import Review from './pages/Review'; import CharacterSelection from './pages/CharacterSelection'; import Character from './pages/Character'; +import Lesson from './pages/Lesson'; const router = createBrowserRouter([ { @@ -49,6 +50,10 @@ const router = createBrowserRouter([ { path: '/character', element: + }, + { + path: '/lesson', + element: } ]) From 498d18892539a819bd9c7086e902af088e7c72a2 Mon Sep 17 00:00:00 2001 From: juanpabloacosta Date: Thu, 30 Nov 2023 07:20:37 -0500 Subject: [PATCH 11/14] Added interfaces and helper function to send request for AI marking. --- frontend/src/logic/sdk.ts | 43 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 3188f8d..92e18b0 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -151,4 +151,45 @@ export async function getAudio(audioId: string): Promise { const blob = await response.blob(); return blob; -} \ No newline at end of file +} + +export interface UserQuestionRequest +{ + question: string; + user_answer: string; + expected: string; + chapter: string; + language: string; +} + +export interface UserQuestionFeedbackResponse +{ + correct: string; + reason: string; +} + +export async function getAIMarking(question: string, user_answer: string, expected: string, chapter: string, language: string): Promise { + const request: UserQuestionRequest = { + question: question, + user_answer: user_answer, + expected: expected, + chapter: chapter, + language: language + }; + + const response = await fetch(`${backendUrl}/ai-mark`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json; +} From a34671434b3701eef40aea82ca2864f1112b2227 Mon Sep 17 00:00:00 2001 From: juanpabloacosta Date: Thu, 30 Nov 2023 07:21:09 -0500 Subject: [PATCH 12/14] Wrote page for Review with one type of review and questions for 1 language. --- frontend/src/pages/Review.tsx | 77 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Review.tsx b/frontend/src/pages/Review.tsx index 99e71fb..e982cad 100644 --- a/frontend/src/pages/Review.tsx +++ b/frontend/src/pages/Review.tsx @@ -1,14 +1,87 @@ +import { useNavigate } from "react-router-dom" import NavBar from "../components/NavBar" +import { getLanguage, getUsername } from "../logic/sdk"; export default function Review() { + const navigate = useNavigate(); + + const writtenReview = ["Questions", "Vocabulary"]; + const verbalReview = ["Questions", "Pronunciation"]; + const language = getLanguage(getUsername()); + + const handleReviewLessonClick = (reviewType: string, lesson: string) => { + type Question = { question: string, wordBank: string[], expected: string, type: string, exercise: string }; + let reviewQuestions: Question[] = []; + if (language === "Japanese") { + reviewQuestions = [ + { + question: 'Translate this sentence: 新幹線で東京に行きます。', + wordBank: ['Explore', 'Train', 'Will', 'City', 'I', 'Go', 'Bullet', 'To', 'Tokyo', 'By', 'Mountain'], + expected: 'I will go to Tokyo by bullet train.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 空港で荷物を受け取ります。', + wordBank: ['Sunset', 'Will', 'Ocean', 'At', 'Adventure', 'Airport', 'The', 'I', 'Receive', 'Luggage'], + expected: 'I will receive luggage at the airport.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: ホテルでチェックインします。', + wordBank: ['Culture', 'At', 'Jungle', 'Will', 'The', 'Check-In', 'Hotel', 'I', 'Historic'], + expected: 'I will check in at the hotel.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 観光名所を訪れます。', + wordBank: ['Desert', 'Mountain', 'Beach', 'Visit', 'Attractions', 'Tourist', 'I', 'Will'], + expected: 'I will visit tourist attractions.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 美味しい地元の食べ物を試します。', + wordBank: ['Try', 'Market', 'Delicious', 'I', 'Food', 'Will', 'River', 'Local', 'Park'], + expected: 'I will try delicious local food.', + type: reviewType, + exercise: lesson + }, + ] + } + navigate('/lesson', { state: { questions: reviewQuestions } }); + } return (
+

Review Page

+

Written

+
+ {writtenReview.map(lesson => ( + + ))} +
+

Verbal/Listening

-

Review Page

+ {verbalReview.map(lesson => ( + + ))}
) - } \ No newline at end of file From 4e8a4acaae35fb0c682ad62d8b6cae0f64a54490 Mon Sep 17 00:00:00 2001 From: juanpabloacosta Date: Thu, 30 Nov 2023 07:21:52 -0500 Subject: [PATCH 13/14] Wrote lesson page (currently only one type of question). --- frontend/src/pages/Lesson.tsx | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 frontend/src/pages/Lesson.tsx diff --git a/frontend/src/pages/Lesson.tsx b/frontend/src/pages/Lesson.tsx new file mode 100644 index 0000000..d830698 --- /dev/null +++ b/frontend/src/pages/Lesson.tsx @@ -0,0 +1,47 @@ +import { useLocation, useNavigate } from 'react-router-dom'; +import { Icon } from '@iconify/react'; +import { useState } from 'react'; +import WrittenQuestionExercise from "../components/WrittenQuestionExercise" + +export default function Course() { + const location = useLocation(); + const navigate = useNavigate(); + const { questions } = location.state; + const [currQuestion, setCurrQuestion] = useState(0); + + const renderLessonContent = () => { + const {question, wordBank, expected, type, exercise} = questions[currQuestion]; + switch (type) { + case 'written': + switch (exercise) { + case 'questions': + return ; + // case 'vocabulary': + // return ; + default: + return null; + } + // case 'verbal-listening': + // switch (exercise) { + // case 'questions': + // return ; + // case 'pronunciation': + // return ; + // default: + // return null; + // } + // default: + // return null; + } + }; + + return ( +
+ navigate(-1)} /> +

Course Page

+
+ {renderLessonContent()} +
+
+ ) +} \ No newline at end of file From 2eab4e97afc5ef4c15f86c1a21918018bb753e33 Mon Sep 17 00:00:00 2001 From: juanpabloacosta Date: Thu, 30 Nov 2023 07:22:13 -0500 Subject: [PATCH 14/14] Wrote component for a WrittenQuestion type exercise. --- .../components/WrittenQuestionExercise.tsx | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 frontend/src/components/WrittenQuestionExercise.tsx diff --git a/frontend/src/components/WrittenQuestionExercise.tsx b/frontend/src/components/WrittenQuestionExercise.tsx new file mode 100644 index 0000000..7a910d2 --- /dev/null +++ b/frontend/src/components/WrittenQuestionExercise.tsx @@ -0,0 +1,103 @@ +import ChatBoxBorder from '../assets/img/chatbox-border.svg'; +import { useState } from 'react'; +import { getAIMarking } from '../logic/sdk'; + +interface WrittenQuestionProps { + question: string; + wordBank: string[]; + expected: string; + chapter: string; + language: string; + setCurrQuestion: Function; +} + +export default function WrittenQuestionExercise({question, wordBank, expected, chapter, language}: WrittenQuestionProps) { + const [selectedWords, setSelectedWords] = useState([]); + const [remainingWords, setRemainingWords] = useState(wordBank); + const lastPunctuation = question[question.length - 1]; + const [answered, setAnswered] = useState(false); + const [correct, setCorrect] = useState(""); + const [reason, setReason] = useState(""); + + const handleWordBankClick = (word: string) => { + setSelectedWords([...selectedWords, word]); + setRemainingWords(remainingWords.filter((w) => w !== word)); + } + + const handleSelectedWordClick = (word: string) => { + setRemainingWords([...remainingWords, word]); + setSelectedWords(selectedWords.filter((w) => w !== word)); + } + + const handleSubmit = () => { + const userAnswer = selectedWords.join(" ") + lastPunctuation; + getAIMarking( + question, + userAnswer, + expected, + chapter, + language + ).then((res) => { + setAnswered(true); + setCorrect(res.correct); + setReason(res.reason); + }) + } + + const ResponseSection = (correct: string | null, reason: string | null) => { + if (!answered) { + return ( +
+
+ {remainingWords.map((word, index) => ( + handleWordBankClick(word)}> + {word} + + ))} +
+ +
+ + ) + } else { + return ( +
+

{correct}

+

{reason}

+
+ ) + } + } + + return ( +
+
+ {question} +
+
+ {selectedWords.map((word, index) => ( + handleSelectedWordClick(word)}> + {word} + + ))} +
+ {ResponseSection(correct, reason)} + {/*
+ {remainingWords.map((word, index) => ( + handleWordBankClick(word)}> + {word} + + ))} +
+ */} +
+ )}