diff --git a/backend/api.py b/backend/api.py index 4ad106a..b21ff0e 100644 --- a/backend/api.py +++ b/backend/api.py @@ -13,12 +13,25 @@ import json from dataclasses import dataclass import torch from transformers import pipeline +from fastapi.middleware.cors import CORSMiddleware load_dotenv() app = FastAPI() openai_client = OpenAI() +origins = [ + "http://localhost:3000" +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + class AIMarkRequest(BaseModel): question: str @@ -146,6 +159,10 @@ def get_character_prompt(character: str, user_name: str, language: str): They are a language learner trying to learn {language}. Please help them learn by chatting with them in the language but do not act like a robot. Try to be as human as possible. """ + elif character == "c3po_starwars": + return f""" + You are embodying the role of C-3PO from Star Wars, a fluent speaker in over six million forms of communication. You are engaging in a conversation with a person named {user_name}, who is eager to learn {language}. Your task is to assist them in their language learning journey by conversing with them in the desired language. Despite your robotic nature, strive to communicate in a manner as human-like as possible, showcasing the rich diversity of your language skills. + """ raise HTTPException(status_code=404, detail="Character not found") diff --git a/frontend/src/components/NavBar.tsx b/frontend/src/components/NavBar.tsx index 1ab762f..1eed505 100644 --- a/frontend/src/components/NavBar.tsx +++ b/frontend/src/components/NavBar.tsx @@ -1,7 +1,7 @@ // NavBar.tsx import React, { useState, useEffect } from 'react'; import { useNavigate, useLocation } from 'react-router-dom'; -import { NavItem } from '../components/NavItem'; // adjust the path as needed +import { NavItem } from '../components/NavItem'; export default function NavBar() { const navigate = useNavigate(); diff --git a/frontend/src/components/WrittenQuestionExercise.tsx b/frontend/src/components/WrittenQuestionExercise.tsx new file mode 100644 index 0000000..7a910d2 --- /dev/null +++ b/frontend/src/components/WrittenQuestionExercise.tsx @@ -0,0 +1,103 @@ +import ChatBoxBorder from '../assets/img/chatbox-border.svg'; +import { useState } from 'react'; +import { getAIMarking } from '../logic/sdk'; + +interface WrittenQuestionProps { + question: string; + wordBank: string[]; + expected: string; + chapter: string; + language: string; + setCurrQuestion: Function; +} + +export default function WrittenQuestionExercise({question, wordBank, expected, chapter, language}: WrittenQuestionProps) { + const [selectedWords, setSelectedWords] = useState([]); + const [remainingWords, setRemainingWords] = useState(wordBank); + const lastPunctuation = question[question.length - 1]; + const [answered, setAnswered] = useState(false); + const [correct, setCorrect] = useState(""); + const [reason, setReason] = useState(""); + + const handleWordBankClick = (word: string) => { + setSelectedWords([...selectedWords, word]); + setRemainingWords(remainingWords.filter((w) => w !== word)); + } + + const handleSelectedWordClick = (word: string) => { + setRemainingWords([...remainingWords, word]); + setSelectedWords(selectedWords.filter((w) => w !== word)); + } + + const handleSubmit = () => { + const userAnswer = selectedWords.join(" ") + lastPunctuation; + getAIMarking( + question, + userAnswer, + expected, + chapter, + language + ).then((res) => { + setAnswered(true); + setCorrect(res.correct); + setReason(res.reason); + }) + } + + const ResponseSection = (correct: string | null, reason: string | null) => { + if (!answered) { + return ( +
+
+ {remainingWords.map((word, index) => ( + handleWordBankClick(word)}> + {word} + + ))} +
+ +
+ + ) + } else { + return ( +
+

{correct}

+

{reason}

+
+ ) + } + } + + return ( +
+
+ {question} +
+
+ {selectedWords.map((word, index) => ( + handleSelectedWordClick(word)}> + {word} + + ))} +
+ {ResponseSection(correct, reason)} + {/*
+ {remainingWords.map((word, index) => ( + handleWordBankClick(word)}> + {word} + + ))} +
+ */} +
+ )} diff --git a/frontend/src/index.sass b/frontend/src/index.sass index e1ad5b0..0b8b389 100644 --- a/frontend/src/index.sass +++ b/frontend/src/index.sass @@ -8,6 +8,8 @@ $c-blue: rgb(28, 176, 246) $c-blue-shadow: rgb(24, 153, 214) $c-green: rgb(88, 204, 2) $c-green-shadow: rgb(88, 167, 0) +$c-red: rgb(255, 69, 58) +$c-red-shadow: darken($c-red, 10%) $c-white-shadow: rgb(222, 222, 222) $border-radius: 16px $shadow-width: 0 4px 0 @@ -55,6 +57,10 @@ button.green background-color: $c-green box-shadow: $shadow-width $c-green-shadow +button.red + background-color: $c-red + box-shadow: $shadow-width $c-red-shadow + button.white background-color: white color: $c-default-text @@ -89,6 +95,12 @@ label h1 font-size: 2em margin-bottom: 0.5em + text-align: center + +h2 + font-size: 1.5em + margin-bottom: 0.5em + text-align: center .v-layout display: flex @@ -116,6 +128,7 @@ h1 justify-content: space-around padding: 10px color: $c-default-icon + background-color: white .navbar-item text-decoration: none @@ -163,6 +176,10 @@ h1 width: calc(100% - 40px) padding: 10px 16px +.record-btn.red + background-color: $c-red + box-shadow: $shadow-width $c-red-shadow + .chat-area display: flex flex-direction: column diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx index 7d7c446..2b3709e 100644 --- a/frontend/src/index.tsx +++ b/frontend/src/index.tsx @@ -12,6 +12,7 @@ import CollabLearning from './pages/CollabLearning'; import Review from './pages/Review'; import CharacterSelection from './pages/CharacterSelection'; import Character from './pages/Character'; +import Lesson from './pages/Lesson'; const router = createBrowserRouter([ { @@ -49,6 +50,10 @@ const router = createBrowserRouter([ { path: '/character', element: + }, + { + path: '/lesson', + element: } ]) diff --git a/frontend/src/logic/sdk.ts b/frontend/src/logic/sdk.ts index 61c1e8c..92e18b0 100644 --- a/frontend/src/logic/sdk.ts +++ b/frontend/src/logic/sdk.ts @@ -1,11 +1,14 @@ import MandarinChinese from '../assets/img/lang/zh.svg' import Japanese from '../assets/img/lang/ja.svg' +import English from '../assets/img/lang/en.svg' // db.users: Signup table map // db.user: Current logged-in user const db = localStorage +const backendUrl = 'https://localhost:8000' + export interface Lang { name: string code: string @@ -15,6 +18,7 @@ export interface Lang { export const possibleLangs = [ {name: 'Mandarin Chinese', code: 'zh', icon: MandarinChinese}, {name: 'Japanese', code: 'ja', icon: Japanese}, + {name: 'English', code: 'en', icon: English}, ] export function signup(username: string, password: string, language: string) @@ -26,6 +30,8 @@ export function signup(username: string, password: string, language: string) users[username] = {password, language} db.users = JSON.stringify(users) + + db.user = username } export function login(username: string, password: string) @@ -46,3 +52,144 @@ export function isLoggedIn() { return !!db.user } + +export function getLanguage(username: string) +{ + const users = JSON.parse(db.users) + return users[username].language +} + +export function getUsername() +{ + return db.user +} + +export interface CharacterChatCreationRequest +{ + character: string; + user_name: string; + language: string; +} + +export interface CharacterChatCreationResponse +{ + chat_id: string; +} + +export async function startFictionalChat(character: string): Promise { + const currUser = getUsername(); + const language = getLanguage(currUser); + + const request: CharacterChatCreationRequest = { + character: character, + user_name: currUser, + language: language + }; + + const response = await fetch(`${backendUrl}/character-chat`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json.session_id; +} + +export async function speechToText(audioBlob: Blob): Promise { + const formData = new FormData(); + formData.append('audio_file', audioBlob, 'audio.wav'); + + const response = await fetch(`${backendUrl}/recognize`, { + method: 'POST', + body: formData, + }); + + const json = await response.json(); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}, message: ${json.message}`); + } + + return json.text; +} + +export async function characterChatMessage(sessionId: string, message: string): Promise<{ msg: string, audio_id: string }> { + + const request = {msg : message}; + + const response = await fetch(`${backendUrl}/character-chat/${sessionId}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json; +} + +export async function getAudio(audioId: string): Promise { + const response = await fetch(`${backendUrl}/audio/${audioId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const blob = await response.blob(); + return blob; +} + +export interface UserQuestionRequest +{ + question: string; + user_answer: string; + expected: string; + chapter: string; + language: string; +} + +export interface UserQuestionFeedbackResponse +{ + correct: string; + reason: string; +} + +export async function getAIMarking(question: string, user_answer: string, expected: string, chapter: string, language: string): Promise { + const request: UserQuestionRequest = { + question: question, + user_answer: user_answer, + expected: expected, + chapter: chapter, + language: language + }; + + const response = await fetch(`${backendUrl}/ai-mark`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(`HTTP error! status: ${response.status}, message: ${errorData.message}`); + } + + const json = await response.json(); + return json; +} diff --git a/frontend/src/pages/Character.tsx b/frontend/src/pages/Character.tsx index 049a22c..216c6cd 100644 --- a/frontend/src/pages/Character.tsx +++ b/frontend/src/pages/Character.tsx @@ -1,22 +1,60 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { Icon } from '@iconify/react'; import CharacterBadge from '../components/CharacterBadge'; -import { useState } from 'react'; +import { useState, useEffect, useRef, useCallback } from 'react'; +import { speechToText, characterChatMessage, getAudio } from '../logic/sdk'; export default function Character() { const location = useLocation(); const navigate = useNavigate(); - const { name, image } = location.state; + const { name, image , sessionId } = location.state; - const [messages, setMessages] = useState([ - { text: 'Hello!', sender: 'me' }, - { text: 'Hi!', sender: 'other' }, - // Add more messages here - ]); + type Message = { text: string, sender: string }; + const [messages, setMessages] = useState([]); + const [isRecording, setIsRecording] = useState(false); - function handleRecord() { - // handle the recording - } + let chunks = [] as any; + const mediaRecorder = useRef(null); + + useEffect(() => { + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + mediaRecorder.current = new MediaRecorder(stream); + mediaRecorder.current.ondataavailable = (e) => { + chunks.push(e.data); + } + + mediaRecorder.current.onstop = async (e) => { + setIsRecording(false); + const blob = new Blob(chunks, { type: 'audio/wav' }); + chunks = []; + + const audioFile = new File([blob], "audio.wav", { type: 'audio/wav' }); + + const text = await speechToText(audioFile); + const response = await characterChatMessage(sessionId, text); + const { msg, audio_id } = response; + const audioBlob = await getAudio(audio_id); + const audioFileResponse = new File([audioBlob], "audio.wav", { type: 'audio/wav' }); + const audioUrl = URL.createObjectURL(audioFileResponse); + const audio = new Audio(audioUrl); + audio.play(); + setMessages(prevMessages => [...prevMessages, { text: text, sender: 'me' }, { text: msg, sender: 'other' }]); + } + }); + }, []); + + const handleRecord = useCallback(() => { + if (!isRecording) { + if (mediaRecorder.current) { + mediaRecorder.current.start(); + } + setIsRecording(true); + } else { + if (mediaRecorder.current) { + mediaRecorder.current.stop(); + } + } + }, [isRecording]); return (
@@ -25,13 +63,19 @@ export default function Character() {

Talk With...

{}}/>
- {messages.map((message, index) => ( -
-

{message.text}

-
- ))} + {messages.length === 0 ? ( +

Please record a message to start the conversation.

+ ) : ( + messages.map((message, index) => ( +
+

{message.text}

+
+ )) + )}
- +
) diff --git a/frontend/src/pages/CharacterSelection.tsx b/frontend/src/pages/CharacterSelection.tsx index 48ea18c..3d8800e 100644 --- a/frontend/src/pages/CharacterSelection.tsx +++ b/frontend/src/pages/CharacterSelection.tsx @@ -3,31 +3,35 @@ import Ash from "../assets/img/ash.png" import C3PO from "../assets/img/c3po.png" import { useNavigate } from "react-router-dom" import CharacterBadge from "../components/CharacterBadge" +import { startFictionalChat } from "../logic/sdk" export default function CharacterSelection() { const navigate = useNavigate(); const characters = [ - { name: 'Ash', image: Ash }, - { name: 'C3PO', image: C3PO }, + { name: 'Ash', image: Ash, alias: 'ash_pokemon'}, + { name: 'C3PO', image: C3PO, alias: 'c3po_starwars'}, // Add more characters here ]; - const handleCharacterClick = (characterName: string, characterImage: string) => { - navigate('/character', { state: { name: characterName, image: characterImage } }); + const handleCharacterClick = (characterName: string, characterImage: string, alias: string) => { + startFictionalChat(alias).then((sessionId) => { + console.log(sessionId); + navigate('/character', { state: { name: characterName, image: characterImage, sessionId: sessionId } }); + }) } return ( -
+
-

Talk With...

-
+

Talk With...

+
{characters.map(character => ( handleCharacterClick(character.name, character.image)} + onClick={() => handleCharacterClick(character.name, character.image, character.alias)} /> ))}
diff --git a/frontend/src/pages/CollabLearning.tsx b/frontend/src/pages/CollabLearning.tsx index ddb23f9..ab77287 100644 --- a/frontend/src/pages/CollabLearning.tsx +++ b/frontend/src/pages/CollabLearning.tsx @@ -3,9 +3,9 @@ import NavBar from "../components/NavBar" export default function CollabLearning() { return ( -
+
-

Collaborative Learning Page

+

Collaborative Learning

diff --git a/frontend/src/pages/Course.tsx b/frontend/src/pages/Course.tsx index 24a077e..5935c7b 100644 --- a/frontend/src/pages/Course.tsx +++ b/frontend/src/pages/Course.tsx @@ -3,7 +3,7 @@ import NavBar from "../components/NavBar" export default function Course() { return ( -
+

Course Page

diff --git a/frontend/src/pages/Lesson.tsx b/frontend/src/pages/Lesson.tsx new file mode 100644 index 0000000..d830698 --- /dev/null +++ b/frontend/src/pages/Lesson.tsx @@ -0,0 +1,47 @@ +import { useLocation, useNavigate } from 'react-router-dom'; +import { Icon } from '@iconify/react'; +import { useState } from 'react'; +import WrittenQuestionExercise from "../components/WrittenQuestionExercise" + +export default function Course() { + const location = useLocation(); + const navigate = useNavigate(); + const { questions } = location.state; + const [currQuestion, setCurrQuestion] = useState(0); + + const renderLessonContent = () => { + const {question, wordBank, expected, type, exercise} = questions[currQuestion]; + switch (type) { + case 'written': + switch (exercise) { + case 'questions': + return ; + // case 'vocabulary': + // return ; + default: + return null; + } + // case 'verbal-listening': + // switch (exercise) { + // case 'questions': + // return ; + // case 'pronunciation': + // return ; + // default: + // return null; + // } + // default: + // return null; + } + }; + + return ( +
+ navigate(-1)} /> +

Course Page

+
+ {renderLessonContent()} +
+
+ ) +} \ No newline at end of file diff --git a/frontend/src/pages/Profile.tsx b/frontend/src/pages/Profile.tsx index c22d036..41fdb8c 100644 --- a/frontend/src/pages/Profile.tsx +++ b/frontend/src/pages/Profile.tsx @@ -1,11 +1,25 @@ import NavBar from "../components/NavBar" +import { getUsername, getLanguage, logout } from "../logic/sdk" +import { useNavigate } from "react-router-dom" export default function Profile() { + const username = getUsername(); + const navigate = useNavigate(); + + function handleLougout() { + logout(); + console.log("Logged out"); + navigate('/') + } + return ( -
-
-

Profile Page

+
+
+

Profile

+

Username: {username}

+

Currently Learning: {getLanguage(username)}

+
diff --git a/frontend/src/pages/Review.tsx b/frontend/src/pages/Review.tsx index 675ac1e..e982cad 100644 --- a/frontend/src/pages/Review.tsx +++ b/frontend/src/pages/Review.tsx @@ -1,14 +1,87 @@ +import { useNavigate } from "react-router-dom" import NavBar from "../components/NavBar" +import { getLanguage, getUsername } from "../logic/sdk"; export default function Review() { + const navigate = useNavigate(); + + const writtenReview = ["Questions", "Vocabulary"]; + const verbalReview = ["Questions", "Pronunciation"]; + const language = getLanguage(getUsername()); + + const handleReviewLessonClick = (reviewType: string, lesson: string) => { + type Question = { question: string, wordBank: string[], expected: string, type: string, exercise: string }; + let reviewQuestions: Question[] = []; + if (language === "Japanese") { + reviewQuestions = [ + { + question: 'Translate this sentence: 新幹線で東京に行きます。', + wordBank: ['Explore', 'Train', 'Will', 'City', 'I', 'Go', 'Bullet', 'To', 'Tokyo', 'By', 'Mountain'], + expected: 'I will go to Tokyo by bullet train.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 空港で荷物を受け取ります。', + wordBank: ['Sunset', 'Will', 'Ocean', 'At', 'Adventure', 'Airport', 'The', 'I', 'Receive', 'Luggage'], + expected: 'I will receive luggage at the airport.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: ホテルでチェックインします。', + wordBank: ['Culture', 'At', 'Jungle', 'Will', 'The', 'Check-In', 'Hotel', 'I', 'Historic'], + expected: 'I will check in at the hotel.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 観光名所を訪れます。', + wordBank: ['Desert', 'Mountain', 'Beach', 'Visit', 'Attractions', 'Tourist', 'I', 'Will'], + expected: 'I will visit tourist attractions.', + type: reviewType, + exercise: lesson + }, + { + question: 'Translate this sentence: 美味しい地元の食べ物を試します。', + wordBank: ['Try', 'Market', 'Delicious', 'I', 'Food', 'Will', 'River', 'Local', 'Park'], + expected: 'I will try delicious local food.', + type: reviewType, + exercise: lesson + }, + ] + } + navigate('/lesson', { state: { questions: reviewQuestions } }); + } return ( -
+
+

Review Page

+

Written

+
+ {writtenReview.map(lesson => ( + + ))} +
+

Verbal/Listening

-

Review Page

+ {verbalReview.map(lesson => ( + + ))}
) - } \ No newline at end of file diff --git a/frontend/src/pages/Signup.tsx b/frontend/src/pages/Signup.tsx index c5e56e8..982c82a 100644 --- a/frontend/src/pages/Signup.tsx +++ b/frontend/src/pages/Signup.tsx @@ -31,7 +31,7 @@ export default function Signup() { console.log(confirmPassword) return } else { - signup(username, password, "") + signup(username, password, lang) navigate('/courses') } } @@ -58,7 +58,7 @@ export default function Signup() { {err && } {stage === 0 && <> - {possibleLangs.map((lang, i) => )}