Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions app/components/UserVideoPane.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { AudioRecorder } from "../../lib/media/audioRecorder";

dotenv.config(); // Load environment variables from .env file

const UserVideoPane = ({ task }) => {
const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => {
const videoRef = useRef(null);
const [mediaStream, setMediaStream] = useState(null);
const [microphonePermissionGranted, setMicrophonePermissionGranted] = useState(false);
Expand Down Expand Up @@ -70,14 +70,14 @@ const UserVideoPane = ({ task }) => {

if (response.ok) {
const jsonResponse = await response.json();
console.log(jsonResponse);
// console.log(jsonResponse);
// Use braces afterwards
const { transcription } = jsonResponse;
console.log("Transcription:", transcription);
// console.log("Transcription:", transcription);
// Do something with the transcription

const concatenatedTranscriptions = transcriptionCaches.slice(lastIndex).join(' ') + transcription;
console.log("Cumulative: " + concatenatedTranscriptions);
// console.log("Cumulative: " + concatenatedTranscriptions);

let prePrompt = "";
for(let i = 0; i < AIResponses.length; i++) {
Expand All @@ -89,7 +89,7 @@ const UserVideoPane = ({ task }) => {
prePrompt += ", ";
}

console.log(prePrompt)
// console.log(prePrompt)

const frequencyCounter = speechEmotions.reduce((counter, emotion) => {
counter[emotion] = (counter[emotion] || 0) + 1;
Expand All @@ -107,7 +107,7 @@ const UserVideoPane = ({ task }) => {
}
}

console.log(contextEmotion);
// console.log(contextEmotion);

const videoFrequencyCounter = videoEmotions.reduce((counter, emotion) => {
counter[emotion] = (counter[emotion] || 0) + 1;
Expand All @@ -125,11 +125,11 @@ const UserVideoPane = ({ task }) => {
}
}

console.log(videoContextEmotion);
// console.log(videoContextEmotion);

// Define the system prompt and user speech
const systemPrompt = "You are a student. A teacher has been tasked with the following: " + task + ". You should ask questions and act confused. Previous conversation: " + prePrompt;
console.log(systemPrompt);
const systemPrompt = taskPrefix + task + taskSuffix + prePrompt;
// console.log(systemPrompt);
const userSpeech = concatenatedTranscriptions + " Context: The user had " + contextEmotion + " as the highest emotion in their speech and " + videoContextEmotion
+ " as the highest emotion in their body language during this current response.";

Expand All @@ -144,7 +144,7 @@ const UserVideoPane = ({ task }) => {
axios.post("/api/ai-response", payload)
.then(response => {
const aiResponse = response.data;
console.log('AI Response:', aiResponse);
// console.log('AI Response:', aiResponse);

setQuestion("GPT: " + aiResponse.assistantReply);
userInputs.push("User: " + concatenatedTranscriptions);
Expand Down Expand Up @@ -339,7 +339,7 @@ const UserVideoPane = ({ task }) => {

if (response.ok) {
const jsonResponse = await response.json();
console.log(jsonResponse);
// console.log(jsonResponse);
// Use braces afterwards
const { transcription } = jsonResponse;

Expand All @@ -348,7 +348,7 @@ const UserVideoPane = ({ task }) => {

// Call the setTranscriptionCaches function to store the transcriptions array in caches
setTranscriptionCaches(transcriptionCaches);
console.log("Transcriptions:", transcriptionCaches);
// console.log("Transcriptions:", transcriptionCaches);
} else {
console.error("Error:", response.status);
// Handle the error
Expand All @@ -366,14 +366,14 @@ const UserVideoPane = ({ task }) => {
const newSocket = new WebSocket(url);

newSocket.onopen = async () => {
console.log('WebSocket connection established');
// console.log('WebSocket connection established');
// Perform any necessary initialization or authentication

recorderRef.current = await AudioRecorder.create();

// Create a closure to capture the current state of `socket`
(async (socket) => {
console.log(socket)
// console.log(socket)
while (socket) {
const blob = await recorderRef.current.record(recordingLengthMs);
// console.log(blob);
Expand All @@ -397,7 +397,7 @@ const UserVideoPane = ({ task }) => {
};

newSocket.onclose = () => {
console.log('WebSocket connection closed -- attempting re-open');
// console.log('WebSocket connection closed -- attempting re-open');
createWebSocketConnection();
// Perform any necessary cleanup or reconnection logic
};
Expand Down Expand Up @@ -472,7 +472,7 @@ const UserVideoPane = ({ task }) => {

return updatedData;
});
console.log(message["prosody"]);
// console.log(message["prosody"]);
}
};

Expand Down
50 changes: 50 additions & 0 deletions app/interview/page.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"use client";

import React, { useEffect, useState, useRef } from 'react';
import dynamic from 'next/dynamic';

const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), {
loading: () => <p>Loading...</p>,
ssr: false, // Disable server-side rendering for this component
});

const InterviewPage = () => {
const taskRef = useRef("");

useEffect(() => {
const url = new URL(window.location.href);
const searchParams = new URLSearchParams(url.search);
let task = searchParams.get('task');

// console.log('task:', task);

if (task == null || task === "") {
taskRef.current = "Explain how to efficiently merge k sorted linked lists.";
} else {
taskRef.current = task;
}
}, []);

const [isComponentLoaded, setIsComponentLoaded] = useState(false);

const taskPrefix = "You are a technical lead in an interview, where the coding question is: ";
const taskSuffix = ". You should delve deep into technical details of their solution, and give constraints when asked. Your goal is to evaluate"
+ " whether hiring them would be beneficial to the company.";

useEffect(() => {
setIsComponentLoaded(true);
}, []);

return (
<>
{isComponentLoaded &&
<div className="w-screen h-full bg-gradient-to-b from-jetBlack-400 to-jetBlack-600">
<h1 className="text-center text-platinum-500 text-3xl font-bold p-4 pt-16">Task: {taskRef.current}</h1>
<DynamicVideoPane taskPrefix={taskPrefix} task={taskRef.current} taskSuffix={taskSuffix} type="interview" />
</div>
}
</>
);
};

export default InterviewPage;
8 changes: 6 additions & 2 deletions app/page.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
"use client";

import Image from 'next/image'
import React, { useState } from 'react';
import Link from 'next/link';
import styles from "./style";
import Navbar from "./components/Navbar";
Expand All @@ -10,9 +14,9 @@ import Pitch from "./components/Pitch";
import Testimony from "./components/Testimony";




export default function Home() {
const [inputValue, setInputValue] = useState('Help a student understand AI double descent.');

return (
<div className="bg-primary w-full overflow-hidden">
<div className={`${styles.paddingX} ${styles.flexCenter}`}>
Expand Down
51 changes: 51 additions & 0 deletions app/pitch/page.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"use client";

import React, { useEffect, useState, useRef } from 'react';
import dynamic from 'next/dynamic';

const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), {
loading: () => <p>Loading...</p>,
ssr: false, // Disable server-side rendering for this component
});

const PitchPage = () => {
const [isComponentLoaded, setIsComponentLoaded] = useState(false);
const taskRef = useRef("");

useEffect(() => {
const url = new URL(window.location.href);
const searchParams = new URLSearchParams(url.search);
let task = searchParams.get('task');

// console.log(url);
// console.log(searchParams);
// console.log('task:', task);

if (task == null || task == "") {
task = "Pitch a startup to empower users' teaching skills by using AI to simulate a learner.";
} else {
taskRef.current = task;
}
}, []);

const taskPrefix = "You are a venture capitalist looking to invest in new firms. A team has come to you to: ";
const taskSuffix = ". You should ask sharp questions about their market, their vision, their growth plan, and"
+ " other aspects of their idea which would be relevant in predicting future commercial success.";

useEffect(() => {
setIsComponentLoaded(true);
}, []);

return (
<>
{isComponentLoaded &&
<div className="w-screen h-full bg-gradient-to-b from-jetBlack-400 to-jetBlack-600">
<h1 className="text-center text-platinum-500 text-3xl font-bold p-4 pt-16">Task: {taskRef.current}</h1>
<DynamicVideoPane taskPrefix={taskPrefix} task={taskRef.current} taskSuffix={taskSuffix} type="pitch" />
</div>
}
</>
);
};

export default PitchPage;
25 changes: 21 additions & 4 deletions app/practice/page.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"use client";

import React, { useEffect, useState } from 'react';
import React, { useEffect, useState, useRef } from 'react';
import dynamic from 'next/dynamic';

const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), {
Expand All @@ -10,7 +10,24 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the

const PracticePage = () => {
const [isComponentLoaded, setIsComponentLoaded] = useState(false);
const task = "Help a student understand double descent and why it is important in Machine Learning.";
const taskRef = useRef("");

useEffect(() => {
const url = new URL(window.location.href);
const searchParams = new URLSearchParams(url.search);
let task = searchParams.get('task');

// console.log('task:', task);

if (task == null || task === "") {
taskRef.current = "Describe the theoretical foundations of double descent and its importance in machine learning.";
} else {
taskRef.current = task;
}
}, []);

const taskPrefix = "You are a student. A teacher has been tasked with the following: ";
const taskSuffix = ". You should ask questions and act confused. Previous conversation: ";

useEffect(() => {
setIsComponentLoaded(true);
Expand All @@ -20,8 +37,8 @@ const PracticePage = () => {
<>
{isComponentLoaded &&
<div className="w-screen h-full bg-gradient-to-b from-jetBlack-400 to-jetBlack-600">
<h1 className="text-center text-platinum-500 text-3xl font-bold p-4 pt-16">Task: {task}</h1>
<DynamicVideoPane task={task} />
<h1 className="text-center text-platinum-500 text-3xl font-bold p-4 pt-16">Task: {taskRef.current}</h1>
<DynamicVideoPane taskPrefix={taskPrefix} task={taskRef.current} taskSuffix={taskSuffix} />
</div>
}
</>
Expand Down
39 changes: 39 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"postcss": "^8.4.24",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-router-dom": "^6.13.0",
"socket.io-client": "^4.6.2",
"stream": "^0.0.2",
"ws": "^8.13.0"
Expand Down
2 changes: 1 addition & 1 deletion pages/api/gpt.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const bufferToStream = (buffer) => {
export default async function handler(req, res) {
if (req.method === 'POST') {
try {
console.log(req.body);
// console.log(req.body);
const base64Audio = req.body;
const audioBuffer = Buffer.from(base64Audio, 'base64');

Expand Down
6 changes: 3 additions & 3 deletions pages/api/websockets.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,22 @@ export default async function handler(req, res) {
});

socket.onopen = () => {
console.log('WebSocket connection established');
// console.log('WebSocket connection established');
// Perform any necessary initialization or authentication
// Send the JSON message to the WebSocket server
socket.send(JSON.stringify(jsonMessage));
};

socket.onmessage = (event) => {
const message = JSON.parse(event.data);
console.log('Received message:', message);
// console.log('Received message:', message);
// Process and handle the received data as needed
// Send the response back to the client
res.status(200).json({ message: 'WebSocket message received' });
};

socket.onclose = () => {
console.log('WebSocket connection closed');
// console.log('WebSocket connection closed');
// Perform any necessary cleanup or reconnection logic
};
} catch (error) {
Expand Down