docs: add simple sound analyser

Author: nartc

apps/kitchen-sink/public/drums-final.mp3

@@ -0,0 +1,79 @@
+import { computed, Injectable, signal } from '@angular/core';
+import { tracks } from './tracks';
+
+declare global {
+	interface Window {
+		webkitAudioContext: typeof AudioContext;
+	}
+}
+
+export type Audio = Awaited<ReturnType<AudioStore['createAudio']>>;
+
+@Injectable()
+export class AudioStore {
+	clicked = signal(false);
+
+	drums = signal<Audio | null>(null);
+	synth = signal<Audio | null>(null);
+	snare = signal<Audio | null>(null);
+
+	loaded = computed(() => {
+		const [drums, synth, snare] = [this.drums(), this.synth(), this.snare()];
+		return !!drums && !!synth && !!snare;
+	});
+
+	constructor() {
+		void Promise.all(
+			tracks.map(({ sound }) => this.createAudio(`./${sound}-final.mp3`).then(this[sound].set.bind(this[sound]))),
+		);
+	}
+
+	start() {
+		const [drums, synth, snare] = [this.drums(), this.synth(), this.snare()];
+		if (!drums || !synth || !snare) return;
+
+		drums.source.start(0);
+		synth.source.start(0);
+		snare.source.start(0);
+		this.clicked.set(true);
+	}
+
+	private async createAudio(url: string) {
+		const response = await fetch(url);
+		const buffer = await response.arrayBuffer();
+		const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+		const analyser = audioContext.createAnalyser();
+		analyser.fftSize = 64;
+		const data = new Uint8Array(analyser.frequencyBinCount);
+		const source = audioContext.createBufferSource();
+		source.buffer = await new Promise((res) => audioContext.decodeAudioData(buffer, res));
+		source.loop = true;
+
+		const gainNode = audioContext.createGain();
+
+		source.connect(analyser);
+		analyser.connect(gainNode);
+
+		let avg = 0;
+
+		return {
+			audioContext,
+			source,
+			data,
+			gainNode,
+			avg,
+			update() {
+				analyser.getByteFrequencyData(data);
+
+				avg = data.reduce((prev, cur) => prev + cur / data.length, 0);
+
+				if (this) {
+					this.avg = avg;
+				}
+
+				// Calculate a frequency average
+				return avg;
+			},
+		};
+	}
+}

apps/kitchen-sink/public/snare-final.mp3

@@ -0,0 +1,33 @@
+import { ChangeDetectionStrategy, Component, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core';
+import { NgtArgs } from 'angular-three';
+import { Track } from './track';
+import { tracks, zoomIndex } from './tracks';
+import { Zoom } from './zoom';
+
+@Component({
+	standalone: true,
+	template: `
+		<ngt-spot-light [position]="[-4, 4, -4]" [angle]="0.06" [penumbra]="1" [castShadow]="true">
+			<ngt-vector2 *args="[2048, 2048]" attach="shadow.mapSize" />
+		</ngt-spot-light>
+
+		@for (track of tracks; track track.sound) {
+			<app-track [sound]="track.sound" [position]="[0, 0, track.positionZ]" [zoom]="zoomIndex() === $index" />
+		}
+
+		<ngt-mesh [receiveShadow]="true" [rotation]="[-Math.PI / 2, 0, 0]" [position]="[0, -0.025, 0]">
+			<ngt-plane-geometry />
+			<ngt-shadow-material [transparent]="true" [opacity]="0.15" />
+		</ngt-mesh>
+	`,
+	schemas: [CUSTOM_ELEMENTS_SCHEMA],
+	changeDetection: ChangeDetectionStrategy.OnPush,
+	imports: [NgtArgs, Track, Zoom],
+	host: { class: 'simple-sound-analyser-soba-experience' },
+})
+export class Experience {
+	protected readonly Math = Math;
+
+	protected readonly tracks = tracks;
+	protected readonly zoomIndex = zoomIndex;
+}

apps/kitchen-sink/public/synth-final.mp3

@@ -0,0 +1,78 @@
+import { ChangeDetectionStrategy, Component, inject } from '@angular/core';
+import { AudioStore } from './audio.store';
+import { tracks, zoomIndex, zoomTrack } from './tracks';
+
+@Component({
+	selector: 'app-overlay',
+	standalone: true,
+	template: `
+		<div
+			[class.cursor-pointer]="audioStore.loaded()"
+			class="absolute left-0 top-0 flex h-screen w-screen flex-col items-center justify-center gap-2 font-mono transition"
+			[class]="audioStore.clicked() ? ['pointer-events-none', 'opacity-0'] : []"
+		>
+			@if (audioStore.loaded()) {
+				<span>This sandbox needs</span>
+				<span>user interaction for audio</span>
+				<strong>(loud audio warning!!!!!)</strong>
+				<button
+					class="cursor-pointer self-center rounded border border-b-4 border-transparent border-b-gray-400 bg-white px-10 py-2"
+					(click)="onClick()"
+				>
+					▶︎
+				</button>
+			} @else {
+				<span>loading</span>
+			}
+		</div>
+		<div
+			class="absolute top-2 left-0 flex w-full items-center justify-between px-2 opacity-0"
+			[class.opacity-100]="audioStore.clicked()"
+		>
+			<div class="flex flex-col gap-2 text-black">
+				<code>
+					Camera currently reacts to:
+					<button
+						class="rounded border border-b-4 border-transparent border-b-gray-400 bg-white px-4 py-1 text-black"
+						(click)="onChangeZoomTrack()"
+					>
+						{{ zoomTrack() }}
+					</button>
+				</code>
+				<code>
+					credits:
+					<a class="underline" href="https://pmndrs.github.io/examples/demos/simple-audio-analyser" target="_blank">
+						R3F Simple Sound Analyser
+					</a>
+				</code>
+			</div>
+			<code class="text-black">Triplet After Triplet · SEGA · Hidenori Shoji</code>
+		</div>
+	`,
+	changeDetection: ChangeDetectionStrategy.OnPush,
+	styles: `
+		:host > div:first-child {
+			background: linear-gradient(15deg, rgb(82, 81, 88) 0%, rgb(255, 247, 248) 100%);
+		}
+	`,
+})
+export class Overlay {
+	protected audioStore = inject(AudioStore);
+	protected readonly zoomTrack = zoomTrack;
+
+	onClick() {
+		if (this.audioStore.loaded()) {
+			this.audioStore.start();
+		}
+	}
+
+	onChangeZoomTrack() {
+		zoomIndex.update((current) => {
+			let nextRandom = Math.floor(Math.random() * tracks.length);
+			while (nextRandom === current) {
+				nextRandom = Math.floor(Math.random() * tracks.length);
+			}
+			return nextRandom;
+		});
+	}
+}

apps/kitchen-sink/src/app/soba/simple-sound-analyser/audio.store.ts

@@ -0,0 +1,28 @@
+import { ChangeDetectionStrategy, Component } from '@angular/core';
+import { NgtCanvas } from 'angular-three';
+import { AudioStore } from './audio.store';
+import { Experience } from './experience';
+import { Overlay } from './overlay';
+
+@Component({
+	standalone: true,
+	template: `
+		<ngt-canvas [sceneGraph]="sceneGraph" [shadows]="true" [camera]="{ position: [-1, 1.5, 2], fov: 25 }" />
+		<app-overlay />
+	`,
+	imports: [NgtCanvas, Overlay],
+	changeDetection: ChangeDetectionStrategy.OnPush,
+	host: { class: 'simple-sound-analyser-soba' },
+	styles: `
+		:host {
+			display: block;
+			height: 100%;
+			width: 100%;
+			background: linear-gradient(15deg, rgb(82, 81, 88) 0%, rgb(255, 247, 248) 100%);
+		}
+	`,
+	providers: [AudioStore],
+})
+export default class SimpleSoundAnalyser {
+	protected sceneGraph = Experience;
+}

apps/kitchen-sink/src/app/soba/simple-sound-analyser/experience.ts

@@ -0,0 +1,79 @@
+import {
+	ChangeDetectionStrategy,
+	Component,
+	computed,
+	CUSTOM_ELEMENTS_SCHEMA,
+	effect,
+	ElementRef,
+	inject,
+	input,
+	viewChild,
+} from '@angular/core';
+import { injectBeforeRender, NgtArgs } from 'angular-three';
+import { NgtsText } from 'angular-three-soba/abstractions';
+import { NgtsPivotControls } from 'angular-three-soba/controls';
+import { InstancedMesh, MeshBasicMaterial, Object3D, PlaneGeometry } from 'three';
+import { AudioStore } from './audio.store';
+
+@Component({
+	selector: 'app-track',
+	standalone: true,
+	template: `
+		<ngt-group [position]="position()">
+			<ngt-instanced-mesh #instanced *args="[undefined, undefined, length()]" [castShadow]="true">
+				<ngt-plane-geometry *args="[0.01, 0.05]" />
+				<ngt-mesh-basic-material [toneMapped]="false" />
+			</ngt-instanced-mesh>
+
+			<ngts-text [text]="sound()" [options]="{ fontSize: 0.05, color: 'black', position: [0.375, 0, 0] }" />
+		</ngt-group>
+	`,
+	schemas: [CUSTOM_ELEMENTS_SCHEMA],
+	changeDetection: ChangeDetectionStrategy.OnPush,
+	imports: [NgtArgs, NgtsText, NgtsPivotControls],
+})
+export class Track {
+	sound = input.required<'drums' | 'synth' | 'snare'>();
+	position = input<[number, number, number]>([0, 0, 0]);
+
+	private instancedRef = viewChild<ElementRef<InstancedMesh<PlaneGeometry, MeshBasicMaterial>>>('instanced');
+
+	private audioStore = inject(AudioStore);
+
+	audio = computed(() => this.audioStore[this.sound()]());
+	protected length = computed(() => {
+		return this.audio()?.data.length;
+	});
+
+	constructor() {
+		effect((onCleanup) => {
+			const audio = this.audio();
+			if (!audio) return;
+			const { gainNode, audioContext } = audio;
+			gainNode.connect(audioContext.destination);
+			onCleanup(() => gainNode.disconnect());
+		});
+
+		const dummy = new Object3D();
+		injectBeforeRender(() => {
+			const instanced = this.instancedRef()?.nativeElement;
+			if (!instanced) return;
+
+			const audio = this.audio();
+			if (!audio) return;
+
+			const { data } = audio;
+			const avg = audio.update();
+
+			// Distribute the instanced planes according to the frequency data
+			for (let i = 0; i < data.length; i++) {
+				dummy.position.set(i * 0.01 * 1.8 - (data.length * 0.01 * 1.8) / 2, data[i] / 2500, 0);
+				dummy.updateMatrix();
+				instanced.setMatrixAt(i, dummy.matrix);
+			}
+			// Set the hue according to the frequency average
+			instanced.material.color.setHSL(avg / 500, 0.75, 0.75);
+			instanced.instanceMatrix.needsUpdate = true;
+		});
+	}
+}

apps/kitchen-sink/src/app/soba/simple-sound-analyser/overlay.ts

@@ -0,0 +1,10 @@
+import { computed, signal } from '@angular/core';
+
+export const tracks = [
+  { sound: 'drums', positionZ: 0.25 },
+  { sound: 'synth', positionZ: -0.25 },
+  { sound: 'snare', positionZ: 0 },
+] as const;
+
+export const zoomIndex = signal(Math.floor(Math.random() * tracks.length));
+export const zoomTrack = computed(() => tracks[zoomIndex()].sound);

apps/kitchen-sink/src/app/soba/simple-sound-analyser/simple-sound-analyser.ts

@@ -0,0 +1,25 @@
+import { Directive, inject, input } from '@angular/core';
+import { injectBeforeRender } from 'angular-three';
+import { PerspectiveCamera } from 'three';
+import { Track } from './track';
+
+@Directive({ standalone: true, selector: 'app-track[zoom]' })
+export class Zoom {
+	enabled = input(false, { alias: 'zoom' });
+
+	track = inject(Track);
+
+	constructor() {
+		injectBeforeRender(({ camera }) => {
+			const enabled = this.enabled();
+			if (!enabled) return;
+
+			const audio = this.track.audio();
+			if (!audio) return;
+
+			// Set the cameras field of view according to the frequency average
+			(camera as PerspectiveCamera).fov = 25 - audio.avg / 15;
+			camera.updateProjectionMatrix();
+		});
+	}
+}

apps/kitchen-sink/src/app/soba/simple-sound-analyser/track.ts

@@ -53,6 +53,10 @@ const routes: Routes = [
 		path: 'inverted-stencil-buffer',
 		loadComponent: () => import('./inverted-stencil-buffer/inverted-stencil-buffer'),
 	},
+	{
+		path: 'simple-sound-analyser',
+		loadComponent: () => import('./simple-sound-analyser/simple-sound-analyser'),
+	},
 	{
 		path: '',
 		redirectTo: 'stars',