audio capture in unity, code clean up

Author: filnow

Assets/Project/Scripts/Player/Actions/HandInteractions.cs

@@ -3,6 +3,8 @@
 using System.IO.MemoryMappedFiles;
 using System.IO;
 using UnityEngine;
+using System.IO.Pipes;
+using System;
 
 public class HandInteractions : MonoBehaviour
 {
@@ -26,10 +28,19 @@ public class HandInteractions : MonoBehaviour
     bool CooldownDrink = false;
 
     private string word;
+    private bool isRecording = false;
+    
+    private int recordingTime = 2; // Set the recording time in seconds
+   
 
     private AudioSource pickUpItemSound;
     private AudioSource putToInventorySound;
     private AudioSource drinkSound;
+    private AudioSource castingSound;
+    private AudioClip recordedClip;
+
+    public float timeToFadeOutPopUp = 1;
+    public float timeOfFadingOutPopUp = 0.007f;
 
 
     private void Awake() //get necessary components
@@ -183,14 +194,12 @@ void CastSpell() //cast spell with SpellCasting class
         {
             if (GameSettings.useSpeech && !PlayerParams.Variables.uiActive) //if using speach then microphone starting to record
             {
-                MemoryMappedFile mmf_word = MemoryMappedFile.OpenExisting("whisper_run");
-                MemoryMappedViewStream stream_word = mmf_word.CreateViewStream();
-                BinaryWriter write_word = new BinaryWriter(stream_word);
-                string noneString = "ok";
-                byte[] noneBytes = System.Text.Encoding.UTF8.GetBytes(noneString);
-                write_word.Write(noneBytes, 0, noneBytes.Length);
-
-                StartCoroutine(PlayerParams.Controllers.spellCasting.WaitForSpell());
+
+                recordedClip = Microphone.Start(null, false, recordingTime, 16000);
+
+                // Wait for the specified recording time
+                StartCoroutine(WaitForRecording());
+
             }
             else if (!PlayerParams.Variables.uiActive) //open spells menu if using speech is off
             {
@@ -199,6 +208,62 @@ void CastSpell() //cast spell with SpellCasting class
         }
     }
 
+    IEnumerator WaitForRecording()
+    {
+        isRecording = true;
+
+        // PopUp cast spell
+        Debug.Log("Whisper listening");
+        FindObjectOfType<HUD>().SpawnPopUp("", "Cast a Spell.", timeToFadeOutPopUp, timeOfFadingOutPopUp);
+        castingSound = FindObjectOfType<SoundManager>().CreateAudioSource(SoundManager.Sound.SFX_CastingSpell);
+        castingSound.Play();
+
+        // Wait for the specified recording time
+        yield return new WaitForSeconds(recordingTime);
+
+        Microphone.End(null);
+
+        byte[] audioData = ConvertAudioClipToByteArray(recordedClip);
+
+        MemoryMappedFile mmf_audio = MemoryMappedFile.OpenExisting("magehand_whisper_audio");
+        MemoryMappedViewStream stream_audio= mmf_audio.CreateViewStream();
+        BinaryWriter write_audio= new BinaryWriter(stream_audio);
+  
+        write_audio.Write(audioData, 0, audioData.Length);
+
+        PlayerParams.Controllers.spellCasting.WriteToMemoryMappedFile("magehand_whisper_run", "ok");
+
+        StartCoroutine(PlayerParams.Controllers.spellCasting.WaitForSpell());
+
+        isRecording = false;
+    }
+
+    byte[] ConvertAudioClipToByteArray(AudioClip clip)
+    {
+        var samples = new float[clip.samples];
+
+        clip.GetData(samples, 0);
+
+        Int16[] intData = new Int16[samples.Length];
+        //converting in 2 float[] steps to Int16[], //then Int16[] to Byte[]
+
+        Byte[] bytesData = new Byte[samples.Length * 2];
+        //bytesData array is twice the size of
+        //dataSource array because a float converted in Int16 is 2 bytes.
+
+        float rescaleFactor = 32767; //to convert float to Int16
+
+        for (int i = 0; i < samples.Length; i++)
+        {
+            intData[i] = (short)(samples[i] * rescaleFactor);
+            Byte[] byteArr = new Byte[2];
+            byteArr = BitConverter.GetBytes(intData[i]);
+            byteArr.CopyTo(bytesData, i * 2);
+        }
+
+        return bytesData;
+    }
+
     void PutDownObject() //put object down to inventory or if in hand is spell then some special interaction
     {
         CooldownPutDown = true;

Assets/Project/Scripts/Player/Actions/SpellCasting.cs

@@ -313,64 +313,60 @@ static string NormalizeTranscribedTextToDisplay(string input)
         return cleanedString;
     }
 
+    public void WriteToMemoryMappedFile(string mapName, string data)
+    {
+        using (MemoryMappedFile mmf = MemoryMappedFile.OpenExisting(mapName))
+        using (MemoryMappedViewStream stream = mmf.CreateViewStream())
+        using (BinaryWriter writer = new BinaryWriter(stream))
+        {
+            byte[] bytes = System.Text.Encoding.UTF8.GetBytes(data);
+            writer.Write(bytes, 0, bytes.Length);
+        }
+    }
+
+    public void ReadFromMemoryMappedFile(string mapName, int bytesNumber, out byte[] frame)
+    {
+        using (MemoryMappedFile mmf = MemoryMappedFile.OpenExisting(mapName))
+        using (MemoryMappedViewStream stream = mmf.CreateViewStream())
+        using (BinaryReader reader = new BinaryReader(stream))
+        {
+            frame = reader.ReadBytes(bytesNumber);
+        }
+    }
+
     public IEnumerator WaitForSpell()
     {
         if (isListening)
         {
-            Debug.Log("Whisper is currently listening - preventing new cast");
+            Debug.Log("Whisper is currently transcrabing - preventing new cast");
             yield break;
         }
 
-        Debug.Log("Whisper listening");
-        FindObjectOfType<HUD>().SpawnPopUp("", "Cast a Spell.", timeToFadeOutPopUp, timeOfFadingOutPopUp);
-        castingSound = FindObjectOfType<SoundManager>().CreateAudioSource(SoundManager.Sound.SFX_CastingSpell);
-        castingSound.Play();
-
         isListening = true;
 
-        MemoryMappedFile mmf_delete = MemoryMappedFile.OpenExisting("whisper");
-        MemoryMappedViewStream stream_delete = mmf_delete.CreateViewStream();
-        BinaryWriter write_delete = new BinaryWriter(stream_delete);
-
-        string noneString = "None";
-        byte[] noneBytes = System.Text.Encoding.UTF8.GetBytes(noneString);
-        write_delete.Write(noneBytes, 0, noneBytes.Length);
+        WriteToMemoryMappedFile("magehand_whisper_text", "None");
 
-        string word = "None";
         string okString = "ok";
 
         while (okString == "ok")
         {
-            MemoryMappedFile mmf_gesture = MemoryMappedFile.OpenExisting("whisper_run");
-            MemoryMappedViewStream stream_gesture = mmf_gesture.CreateViewStream();
-            BinaryReader reader_gesture = new BinaryReader(stream_gesture);
-            byte[] frameGesture = reader_gesture.ReadBytes(2);
-
+            byte[] frameGesture;
+            ReadFromMemoryMappedFile("magehand_whisper_run", 2, out frameGesture);
             okString = System.Text.Encoding.UTF8.GetString(frameGesture, 0, 2);
             //Debug.Log(word);
             yield return new WaitForFixedUpdate();
         }
 
-        MemoryMappedFile mmf_word = MemoryMappedFile.OpenExisting("whisper");
-        MemoryMappedViewStream stream_word = mmf_word.CreateViewStream();
-        BinaryReader read_word = new BinaryReader(stream_word);
-
-        byte[] frame = read_word.ReadBytes(10);
+        byte[] frameWord;
+        ReadFromMemoryMappedFile("magehand_whisper_text", 10, out frameWord);
 
-        word = System.Text.Encoding.UTF8.GetString(frame, 0, 10).Split(";")[0];
+        string word = System.Text.Encoding.UTF8.GetString(frameWord).Split(";")[0];
         Debug.Log("Whisper transcribed word: " + word);
+
         FindObjectOfType<HUD>().SpawnPopUp("", "Casting word:<br>" + word, timeToFadeOutPopUp, timeOfFadingOutPopUp, false);
         Destroy(castingSound);
 
-
-        MemoryMappedFile mmf_run = MemoryMappedFile.OpenExisting("whisper_run");
-        MemoryMappedViewStream stream_run = mmf_run.CreateViewStream();
-        BinaryWriter write_run = new BinaryWriter(stream_run);
-
-        string runString = "no";
-        byte[] runBytes = System.Text.Encoding.UTF8.GetBytes(runString);
-        write_run.Write(runBytes, 0, runBytes.Length);
-
+        WriteToMemoryMappedFile("magehand_whisper_run", "no");
 
         CastSpellFromName(word);
 

Assets/StreamingAssets/Whisper/_internal/models/models--Systran--faster-whisper-tiny/blobs.meta

@@ -3,17 +3,10 @@
 import sys
 
 import numpy as np
-import pyaudio
 from faster_whisper import WhisperModel
 from multiprocessing.shared_memory import SharedMemory
 
 
-STEP_IN_SEC: int = 5
-LENGHT_IN_SEC: int = 2    
-NB_CHANNELS = 1
-RATE = 16000
-CHUNK = RATE
-
 WHISPER_LANGUAGE = "en"
 WHISPER_THREADS = 2
 
@@ -24,32 +17,19 @@
 
 whisper = WhisperModel("tiny", device="cpu", compute_type="int8", cpu_threads=WHISPER_THREADS, download_root=task_file_path, local_files_only=True)
 
-shared_mem_whisper = SharedMemory(name='whisper', create=True, size=15)
-shared_mem_run = SharedMemory(name='whisper_run', create=True, size=2)
+shared_mem_whisper = SharedMemory(name='magehand_whisper_text', create=True, size=15)
+shared_mem_run = SharedMemory(name='magehand_whisper_run', create=True, size=2)
+shared_mem_audio  = SharedMemory(name='magehand_whisper_audio', create=True, size=64000)
 
 run = 'no'
 shared_mem_run.buf[:2] = bytearray(run.encode('utf-8'))
 
 running = 'None' + ';' + ('a' * (9 - len('None')))
 shared_mem_whisper.buf[:len(running)] = bytearray(running.encode('utf-8'))
 
-audio = pyaudio.PyAudio()
-stream = audio.open(
-    format=pyaudio.paInt16,
-    channels=NB_CHANNELS,
-    rate=RATE,
-    input=True,
-    frames_per_buffer=CHUNK,    
-    )
-
 while True:
     if shared_mem_run.buf[:2].tobytes().decode('utf-8') == "ok":
-       
-        audio_data = b""
-        for _ in range(STEP_IN_SEC):
-            chunk = stream.read(RATE)    
-            audio_data += chunk
-
+        audio_data = shared_mem_audio.buf[:64000].tobytes()
         audio_data_array: np.ndarray = np.frombuffer(audio_data, np.int16).astype(np.float32) / 255.0
 
         segments, _ = whisper.transcribe(audio_data_array,