Feature/singing UI

.rubocop.yml

@@ -43,6 +43,9 @@ Layout/LineLength:
 Style/Lambda:
   EnforcedStyle: lambda
 
+Layout/SpaceAroundOperators:
+  EnforcedStyleForExponentOperator: space
+
 Style/RedundantReturn:
   Enabled: false
 

Gemfile.lock

@@ -2,6 +2,7 @@ PATH
   remote: .
   specs:
     fet (0.3.1)
+      ffi (~> 1.15, >= 1.15.4)
       gli (~> 2.20, >= 2.20.1)
       ice_nine (~> 0.11.2)
       midilib (~> 2.0, >= 2.0.5)
@@ -17,6 +18,7 @@ GEM
       simplecov (>= 0.15, < 0.22)
     coderay (1.1.3)
     docile (1.4.0)
+    ffi (1.15.4)
     gli (2.20.1)
     ice_nine (0.11.2)
     method_source (1.0.0)

README.md

@@ -19,6 +19,26 @@ brew install timidity
 apt install timidity
 ```
 
+Install sox for audio recording.
+#### OS X
+```sh
+brew install sox
+```
+#### Ubuntu
+```sh
+apt install sox
+```
+
+Install dependencies for pitch detection.
+#### OS X
+```sh
+brew install armadillo mlpack
+```
+#### Ubuntu
+```sh
+apt install libarmadillo-dev libmlpack-dev
+```
+
 ### Gem
 Add this line to your application's Gemfile:
 

fet.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Gem runtime dependencies - place development dependencies inside Gemfile
+  spec.add_dependency "ffi", "~> 1.15", ">= 1.15.4"
   spec.add_dependency "gli", "~> 2.20", ">= 2.20.1"
   spec.add_dependency "ice_nine", "~> 0.11.2"
   spec.add_dependency "midilib", "~> 2.0", ">= 2.0.5"

lib/fet.rb

@@ -3,11 +3,11 @@
 require "ice_nine"
 require "ice_nine/core_ext/object"
 
-Dir["#{__dir__}/fet/**/*.rb"].each { |file| require_relative(file.delete_prefix("#{__dir__}/")) }
-
 # Base Gem module
 module Fet
   def self.root
     File.expand_path("..", __dir__)
   end
 end
+
+Dir["#{__dir__}/fet/**/*.rb"].each { |file| require_relative(file.delete_prefix("#{__dir__}/")) }

lib/fet/degree.rb

@@ -25,6 +25,21 @@ def initialize(degree_name)
       validate_degree_name!
     end
 
+    def self.from_degree_index(degree_index, accidental_type:)
+      degree_names = DEGREE_NAMES[degree_index]
+      degree_name = if degree_names.size == 1
+                      degree_names[0]
+                    else
+                      case accidental_type
+                      when "#"
+                        degree_names[0]
+                      when "b"
+                        degree_names[1]
+                      end
+                    end
+      return new(degree_name)
+    end
+
     def degree_accidental
       return degree_name.size == 2 ? degree_name[0] : nil
     end

lib/fet/degrees.rb

@@ -23,6 +23,11 @@ def degree_names_of_midi_value(midi_value)
       return Degree::DEGREE_NAMES[degree_index_of_midi_value(midi_value)]
     end
 
+    def note_names_of_midi_value(midi_value)
+      degree_names = degree_names_of_midi_value(midi_value)
+      return degree_names.map { |degree_name| note_name_of_degree(degree_name) }
+    end
+
     def degree_index_of_midi_value(midi_value)
       return MidiNote.new(midi_value).degree(root_midi_value)
     end

lib/fet/frequency.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "midi_note"
+
+module Fet
+  # Class responsible for calculating frequencies of a note and vice versa
+  class Frequency
+    # REFERENCE: https://en.wikipedia.org/wiki/Piano_key_frequencies
+    ZEROTH_OCTAVE_MIDI_VALUE_TO_FREQUENCY = {
+      MidiNote.from_note("C", 0).midi_value => 16.35160,
+      MidiNote.from_note("Db", 0).midi_value => 17.32391,
+      MidiNote.from_note("D", 0).midi_value => 18.35405,
+      MidiNote.from_note("Eb", 0).midi_value => 19.44544,
+      MidiNote.from_note("E", 0).midi_value => 20.60172,
+      MidiNote.from_note("F", 0).midi_value => 21.82676,
+      MidiNote.from_note("Gb", 0).midi_value => 23.12465,
+      MidiNote.from_note("G", 0).midi_value => 24.49971,
+      MidiNote.from_note("Ab", 0).midi_value => 25.95654,
+      MidiNote.from_note("A", 0).midi_value => 27.50000,
+      MidiNote.from_note("Bb", 0).midi_value => 29.13524,
+      MidiNote.from_note("B", 0).midi_value => 30.86771,
+    }.deep_freeze
+
+    # NOTE: the frequency of a note an octave above is 2 times the frequency of the octave below
+    MIDI_VALUE_TO_FREQUENCY = (0..10).map do |octave_number|
+      ZEROTH_OCTAVE_MIDI_VALUE_TO_FREQUENCY.map do |midi_value, frequency|
+        [midi_value + (12 * octave_number), frequency * (2 ** octave_number)]
+      end
+    end.flatten(1).to_h.deep_freeze
+
+    MIDI_VALUES = MIDI_VALUE_TO_FREQUENCY.keys.deep_freeze
+    FREQUENCIES = MIDI_VALUE_TO_FREQUENCY.values.deep_freeze
+
+    # NOTE: this divides the frequencies into equal buckets with distance of 1 between them
+    # NOTE: round to 3 decimal places: we only care about dividing notes into 100 equal buckets, additional decimal place allows for rounding
+    # REFERENCE: https://en.wikipedia.org/wiki/Equal_temperament
+    TWELFTH_ROOT_OF_TWO = 2 ** (1.0 / 12.0)
+    def self.frequency_to_bucket_value(frequency)
+      rounded_value = Math.log(frequency, TWELFTH_ROOT_OF_TWO).round(3)
+      return Rational((rounded_value * 1000).round, 1000)
+    end
+
+    FREQUENCY_LOGARITHMS = FREQUENCIES.map do |frequency|
+      frequency_to_bucket_value(frequency)
+    end.deep_freeze
+
+    # This function returns the frequency of the given midi value, e.g 123
+    def self.midi_value_to_frequency(midi_value)
+      return MIDI_VALUE_TO_FREQUENCY[midi_value]
+    end
+
+    # This function returns the frequency of the given note, e.g. E8
+    def self.note_to_frequency(note_name, octave_number)
+      return midi_value_to_frequency(MidiNote.from_note(note_name, octave_number).midi_value)
+    end
+
+    # This function returns:
+    # 1) the midi value the frequency represents and
+    # 2) the offset (in cents) for the given frequency in range [-50, 50)
+    def self.frequency_to_midi_value(frequency)
+      frequency_bucket_value = frequency_to_bucket_value(frequency)
+      difference = frequency_bucket_value - FREQUENCY_LOGARITHMS[0]
+      midi_index = difference.round
+      cents = ((difference - midi_index) * 100).round
+      midi_value = MIDI_VALUES[midi_index]
+      if cents == 50
+        midi_value += 1
+        cents -= 100
+      end
+      return midi_value, cents
+    end
+  end
+end

lib/fet/pitch_detector.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# NOTE: To attach function, you need to know its symbol, which can be determined like so:
+#       nm -gU /usr/local/lib/libpitch_detection.so (or whatever shared library object)
+# Other references:
+# https://github.com/ffi/ffi/issues/554
+# https://stackoverflow.com/questions/32904417/how-to-wrap-stdvector-of-custom-structure-with-ffi
+# https://stackoverflow.com/questions/29389334/how-do-i-handle-ruby-arrays-in-ruby-ffi-gem
+
+require "ffi"
+
+module Fet
+  class PitchDetector
+    extend FFI::Library
+
+    # NOTE: passing array will load one of them or fail - used to provide .so and .dylib
+    # TODO: perhaps delay ffi_lib call until it is required (which is only during the singing exercises),
+    #       otherwise if the user doesn't have the required dependencies for this library, it will fail to even run the console
+    ffi_lib([File.join(Fet.root, "ext", "fet", "libpitch_detection.dylib"), File.join(Fet.root, "ext", "fet", "libpitch_detection.so")])
+
+    attach_function :yin, "_ZN5pitch3yinEPKdmi", [:pointer, :size_t, :int], :double
+    attach_function :mpm, "_ZN5pitch3mpmEPKdmi", [:pointer, :size_t, :int], :double
+    attach_function :pyin, "_ZN5pitch4pyinEPKdmi", [:pointer, :size_t, :int], :double
+    attach_function :pmpm, "_ZN5pitch4pmpmEPKdmi", [:pointer, :size_t, :int], :double
+    attach_function :swipe, "_ZN5pitch5swipeEPKdmi", [:pointer, :size_t, :int], :double
+
+    def self.frequency(samples, sample_rate)
+      FFI::MemoryPointer.new(:double, samples.size) do |pointer|
+        pointer.put_array_of_double(0, samples)
+        result = yin(pointer, samples.size, sample_rate)
+        return nil if result.negative?
+
+        return result
+      end
+    end
+  end
+end

lib/fet/sox_interface.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Fet
+  # Class responsible for interfacing with sox
+  class SoxInterface
+    def initialize
+      self.frequency_queue = Queue.new
+      self.recording_thread = nil
+      self.sample_rate = 44_100
+      # NOTE: analyze 0.1 seconds
+      self.buffer_size = sample_rate / 10
+    end
+
+    def read_frequency
+      return frequency_queue.pop
+    end
+
+    def start_recording
+      Thread.abort_on_exception = true
+      self.recording_thread = Thread.new do
+        # NOTE: Record with 44.1K sample rate (-r) to a single mono channel (-c) and output
+        #       the amplitudes of the wave file as 32-bit signed integers (-t) to STDOUT (-)
+        #       and disregard STDERR. Mono recording is required for pitch detection to work.
+        # REFERENCE: https://stackoverflow.com/questions/1154846/continuously-read-from-stdout-of-external-process-in-ruby
+        # REFERENCE: https://stackoverflow.com/questions/43208040/ruby-continuously-output-stdout-of-a-long-running-shell-command
+        # REFERENCE: https://stackoverflow.com/questions/7212573/when-to-use-each-method-of-launching-a-subprocess-in-ruby
+        # REFERENCE: https://stackoverflow.com/questions/42541588/any-way-i-can-get-sox-to-just-print-the-amplitude-values-from-a-wav-file
+        # NOTE: PTY.spawn doesn't work because it converts \n to \r\n for some reason, which is bad when outputting a binary file.
+        IO.popen("rec -r #{sample_rate} -c 1 -t s32 - 2>/dev/null") do |stdout|
+          loop do
+            frequency_queue.push(calculate_frequency(stdout))
+            # break if stdout.eof?
+          end
+        end
+      end
+    end
+
+    private
+
+    attr_accessor :frequency_queue, :recording_thread, :sample_rate, :buffer_size
+
+    def calculate_frequency(io)
+      # NOTE: I originally wanted to use the "wavefile" gem to analyze the wav file; however, it doesn't work with pipes because it uses
+      #       IO#seek (doesn't work on pipes). So instead going to manually extract the amplitude values.
+      samples = []
+      buffer_size.times do
+        # break if io.eof?
+        # NOTE: unpacking with "l" means converting a string that looks like "\x00\xdf\xa0..." to "32-bit signed, native endian (int32_t)"
+        samples << io.read(4).unpack1("l")
+      end
+      result = Fet::PitchDetector.frequency(samples, sample_rate)
+      result = filter_frequency_by_range(result)
+      return result
+    end
+
+    # Vocal range is E2-C6 => 80Hz-1100Hz
+    # REFERENCE: https://en.wikipedia.org/wiki/List_of_basses_in_non-classical_music
+    # REFERENCE: https://en.wikipedia.org/wiki/List_of_sopranos_in_non-classical_music
+    # REFERENCE: https://en.wikipedia.org/wiki/Scientific_pitch_notation
+    # NOTE: I was also thinking of filtering by amplitude, but that doesn't seem
+    #       to be necessary from my tests.
+    def filter_frequency_by_range(frequency)
+      return nil if frequency.nil?
+      return nil if frequency < 80 || frequency > 1100
+
+      return frequency
+    end
+  end
+end

lib/fet/ui/color_scheme.rb

@@ -8,6 +8,7 @@ module ColorScheme
       WHITE = "#FAF9F0".deep_freeze
       RED = "#931621".deep_freeze
       GREEN = "#2F754E".deep_freeze
+      ORANGE = "#FE9000".deep_freeze
     end
   end
 end

lib/fet/ui/game.rb

@@ -7,6 +7,7 @@
 require_relative "level"
 require_relative "score"
 require_relative "timer"
+require_relative "../sox_interface"
 
 module Fet
   module Ui
@@ -15,7 +16,7 @@ class Game
       include GameSetupHelper
       include GameLoopHandler
 
-      attr_accessor :level, :score, :timer, :note_range, :tmp_directory,
+      attr_accessor :level, :score, :timer, :note_range, :tmp_directory, :recorder,
                     :tempo, :number_of_degrees, :key_type, :next_on_correct, :limit_degrees
 
       def initialize(tempo:, degrees:, key_type:, next_on_correct:, limit_degrees: [])
@@ -26,6 +27,7 @@ def initialize(tempo:, degrees:, key_type:, next_on_correct:, limit_degrees: [])
         self.next_on_correct = next_on_correct
         self.limit_degrees = limit_degrees
         self.tmp_directory = Dir.mktmpdir
+        self.recorder = SoxInterface.new
         initialize_ui_objects
         validate!
         setup_window
@@ -35,6 +37,7 @@ def start
         score.start
         level.start
         timer.start
+        recorder.start_recording
         show_window
         Fet::ScoreSummary.add_entry(self)
       end

lib/fet/ui/game_loop_handler.rb

@@ -10,6 +10,8 @@ def handle_update_loop
         score.handle_update_loop
         level.handle_update_loop
         timer.handle_update_loop
+
+        handle_custom_events
       end
 
       def handle_event_loop(event)

lib/fet/ui/level.rb

@@ -49,19 +49,25 @@ def answered_correctly?
 
       attr_accessor :midi_values, :full_question_music, :chord_progression_music, :notes_music
 
+      # TODO: to allow for single-note listening exercises, the only change required is to set self.midi_values = [SOME_VALUE] here
       def start_self
         self.question_number += 1
         self.degrees = generate_degrees
         self.midi_values = degrees.select_degrees_from_midi_values(game.note_range, game.number_of_degrees, game.limit_degrees)
 
         update_music_objects
-        play_full_question
+        # play_full_question # for listening exercises
+        play_chord_progression # for singing exercises
       end
 
       def play_full_question
         full_question_music.play
       end
 
+      def play_chord_progression
+        chord_progression_music.play
+      end
+
       def generate_degrees
         root_midi_values = game.key_type == "major" ? Fet::MAJOR_ROOT_MIDI_VALUES : Fet::MINOR_ROOT_MIDI_VALUES
         root_name, root_midi_value = root_midi_values.to_a.sample

lib/fet/ui/level_loop_handler.rb

@@ -14,7 +14,9 @@ def handle_event_loop(event)
         note_boxes.handle_event_loop(event)
       end
 
-      def handle_update_loop; end
+      def handle_update_loop
+        note_boxes.handle_update_loop
+      end
 
       private