Av sync issue - restart play when FPS fluctuates and returns to normal

src/main/js/webrtc_adaptor.js

@@ -387,6 +387,32 @@
 		if (this.initializeComponents) {
 			this.initialize();
 		}
+
+		/**
+		 * Automatic AV Sync Recovery Configurations
+		 */
+		this.autoResyncOnFrameDrop = initialValues.autoResyncOnFrameDrop ?? false;
+		this.autoResyncCooldownMs = initialValues.autoResyncCooldownMs ?? 10000;
+		this._lastAutoResyncTime = {}; // { streamId: timestamp }
+		// FPS fluctuation-based auto-resync config
+		this.fpsFluctuationWindowSize = initialValues.fpsFluctuationWindowSize ?? 3; // e.g., last 5 seconds
+		this.fpsDropPercentThreshold = initialValues.fpsDropPercentThreshold ?? 0.1; // 10% drop
+		this.fpsFluctuationStdDevThreshold = initialValues.fpsFluctuationStdDevThreshold ?? 1; // e.g., 5 FPS
+		this.fpsFluctuationConsecutiveCount = initialValues.fpsFluctuationConsecutiveCount ?? 1;
+		this._fpsHistory = {}; // { streamId: [fps] }
+		this._fpsFluctuationCount = {}; // { streamId: count }
+		this._lastFramesReceived = {}; // { streamId: last framesReceived }
+		this._lastStatsTime = {}; // { streamId: last timestamp }
+		// Improved fluctuation handling
+		this.fpsStableStdDevThreshold = initialValues.fpsStableStdDevThreshold ?? 2;
+		// Dynamic healthy FPS baseline and stabilization window
+		this.fpsHealthyBaselineWindow = initialValues.fpsHealthyBaselineWindow ?? 5; // samples for healthy baseline
+		this.fpsStableWindow = initialValues.fpsStableWindow ?? 3; // samples for stabilization
+		this.fpsStablePercentOfBaseline = initialValues.fpsStablePercentOfBaseline ?? 0.8; // 80%
+		this._fpsHealthyBaseline = {}; // { streamId: [fps] }
+		this._lastHealthyAvgFps = {}; // { streamId: number }
+		this.healthyWindowSize = initialValues.healthyWindowSize ?? 5;
+		this._fpsFluctuating = {}; // { streamId: boolean }
 	}
 
 	/**
@@ -1744,10 +1770,100 @@
 		this.remotePeerConnectionStats[streamId].audioPacketsReceived = audioPacketsReceived;
 		this.remotePeerConnectionStats[streamId].videoPacketsReceived = videoPacketsReceived;
 
+		const now = Date.now();
+		const lastFrames = this._lastFramesReceived[streamId] ?? framesReceived;
+		const lastTime = this._lastStatsTime[streamId] ?? now;
+		const calculatedFps = (framesReceived - lastFrames) / ((now - lastTime) / 1000);
+		this._lastFramesReceived[streamId] = framesReceived;
+		this._lastStatsTime[streamId] = now;
+
+		this.checkAndHandleFpsFluctuation(streamId, calculatedFps, now);
+
 		return this.remotePeerConnectionStats[streamId];
 	}
 
+	checkAndHandleFpsFluctuation(streamId, calculatedFps, now) {
+		if (!this._fpsHistory[streamId]) this._fpsHistory[streamId] = [];
+		this._fpsHistory[streamId].push(calculatedFps);
+		if (this._fpsHistory[streamId].length > this.fpsFluctuationWindowSize + 1) {
+			this._fpsHistory[streamId].shift();
+		}
+
+		if (this._fpsHistory[streamId].length > this.fpsFluctuationWindowSize) {
+			Logger.debug(`FPS Fluctuation Detection - Stream: ${streamId}, Current FPS: ${calculatedFps.toFixed(2)}`);
+			// Exclude the current FPS from the average of previous values
+			const prevFpsValues = this._fpsHistory[streamId].slice(0, -1);
+			const prevAvgFps = prevFpsValues.reduce((a, b) => a + b, 0) / prevFpsValues.length;
+			const prevStdDev = Math.sqrt(prevFpsValues.reduce((a, b) => a + Math.pow(b - prevAvgFps, 2), 0) / prevFpsValues.length);
+
+			// Update healthy baseline and last healthy avg if not fluctuating
+			if (!this._fpsFluctuating[streamId]) {
+				if (!this._fpsHealthyBaseline[streamId]) this._fpsHealthyBaseline[streamId] = [];
+				this._fpsHealthyBaseline[streamId].push(calculatedFps);
+				if (this._fpsHealthyBaseline[streamId].length > this.healthyWindowSize) {
+					this._fpsHealthyBaseline[streamId].shift();
+				}
+				// Update last healthy avg
+				const healthyArr = this._fpsHealthyBaseline[streamId];
+				if (healthyArr.length === this.healthyWindowSize) {
+					this._lastHealthyAvgFps[streamId] = healthyArr.reduce((a, b) => a + b, 0) / healthyArr.length;
+				}
+			}
 
+			// Fluctuation detection: use last healthy avg
+			const healthyAvg = this._lastHealthyAvgFps[streamId] ?? calculatedFps;
+			const fluctuationDetected = (
+				calculatedFps < healthyAvg * (1 - this.fpsDropPercentThreshold) &&
+				prevStdDev > this.fpsFluctuationStdDevThreshold
+			);
+			if (fluctuationDetected) {
+				this._fpsFluctuationCount[streamId] = (this._fpsFluctuationCount[streamId] || 0) + 1;
+				Logger.debug(`FPS Fluctuation Detection - Stream: ${streamId}, Current FPS: ${calculatedFps.toFixed(2)}, Count: ${this._fpsFluctuationCount[streamId]}`);
+				if (this._fpsFluctuationCount[streamId] >= this.fpsFluctuationConsecutiveCount) {
+					this._fpsFluctuating[streamId] = true;
+				}
+			} else {
+				this._fpsFluctuationCount[streamId] = 0;
+			}
+
+			// If we are in a fluctuating state, check for stabilization
+			if (this._fpsFluctuating[streamId]) {
+				// Use only the last N samples for stabilization
+				const stableWindow = this._fpsHistory[streamId].slice(-this.fpsStableWindow);
+				const stableAvg = stableWindow.reduce((a, b) => a + b, 0) / stableWindow.length;
+				const stableStdDev = Math.sqrt(stableWindow.reduce((a, b) => a + Math.pow(b - stableAvg, 2), 0) / stableWindow.length);
+				// Use the healthy baseline
+				const healthyBaselineArr = this._fpsHealthyBaseline[streamId] || [];
+				const healthyBaseline = healthyBaselineArr.length > 0 ? (healthyBaselineArr.reduce((a, b) => a + b, 0) / healthyBaselineArr.length) : 0;
+				Logger.debug(`FPS Stabilization Check - Stream: ${streamId}, StableAvg: ${stableAvg.toFixed(2)}, StableStdDev: ${stableStdDev.toFixed(2)}, HealthyBaseline: ${healthyBaseline.toFixed(2)}`);
+				if (stableStdDev < this.fpsStableStdDevThreshold && stableAvg > healthyBaseline * this.fpsStablePercentOfBaseline) {
+					// Now stable, trigger restart
+					if (this.autoResyncOnFrameDrop && this.playStreamId && this.playStreamId.includes(streamId)) {
+						if (!this._lastAutoResyncTime[streamId] || now - this._lastAutoResyncTime[streamId] > this.autoResyncCooldownMs) {
+							this._lastAutoResyncTime[streamId] = now;
+							Logger.warn(`Auto-resync triggered for stream ${streamId} after FPS stabilized. StableAvg: ${stableAvg.toFixed(2)}, StableStdDev: ${stableStdDev.toFixed(2)}, HealthyBaseline: ${healthyBaseline.toFixed(2)}`);
+							this.notifyEventListeners("auto_resync_triggered", { streamId, calculatedFps, stableAvg, stableStdDev, healthyBaseline, stabilized: true });
+							this.stop(streamId);
+							setTimeout(() => {
+								this.play(
+									streamId,
+									this.playToken,
+									this.playRoomId,
+									this.playEnableTracks,
+									this.playSubscriberId,
+									this.playSubscriberCode,
+									this.playMetaData,
+									this.playRole
+								);
+							}, 500);
+						}
+					}
+					this._fpsFluctuating[streamId] = false; // Reset
+					this._fpsFluctuationCount[streamId] = 0;
+				}
+			}
+		}
+	}
 
 	/**
 	 * Called to start a periodic timer to get statistics periodically (5 seconds) for a specific stream.

src/test/js/webrtc_adaptor.test.js

@@ -2101,6 +2101,140 @@ describe("WebRTCAdaptor", function() {
     		expect(initPeerConnection.calledWithExactly(streamId, "play")).to.be.true;
     	});
 
+	describe("checkAndHandleFpsFluctuation", function() {
+		let adaptor;
+		let streamId = "testStream";
+		let now = Date.now();
+		let loggerDebugStub, loggerWarnStub;
+		let stopStub, playStub;
+		let notifyEventListenersStub;
 
+		beforeEach(function() {
+			adaptor = new WebRTCAdaptor({
+				websocketURL: "ws://example.com",
+				initializeComponents: false,
+			});
+			// Set up stubs for Logger
+			loggerDebugStub = sinon.stub(window.log, "debug");
+			loggerWarnStub = sinon.stub(window.log, "warn");
+			// Set up stubs for stop/play and event notification
+			stopStub = sinon.stub(adaptor, "stop");
+			playStub = sinon.stub(adaptor, "play");
+			notifyEventListenersStub = sinon.stub(adaptor, "notifyEventListeners");
+			// Set up playStreamId for auto-resync
+			adaptor.playStreamId = [streamId];
+			// Set up thresholds for easier testing
+			adaptor.fpsFluctuationWindowSize = 2;
+			adaptor.fpsFluctuationStdDevThreshold = 1;
+			adaptor.fpsDropPercentThreshold = 0.1;
+			adaptor.fpsFluctuationConsecutiveCount = 2;
+			adaptor.fpsStableWindow = 2;
+			adaptor.fpsStableStdDevThreshold = 1;
+			adaptor.fpsStablePercentOfBaseline = 0.8;
+			adaptor.healthyWindowSize = 2;
+			adaptor.autoResyncOnFrameDrop = true;
+			adaptor.autoResyncCooldownMs = 1000;
+		});
+
+		afterEach(function() {
+			sinon.restore();
+		});
+
+		it("should update healthy baseline and not trigger fluctuation or resync on stable FPS", function() {
+			adaptor._fpsFluctuating = {};
+			adaptor._fpsHealthyBaseline = {};
+			adaptor._lastHealthyAvgFps = {};
+			adaptor._fpsHistory = {};
+			adaptor._fpsFluctuationCount = {};
+			adaptor._lastAutoResyncTime = {};
+			// Push stable FPS values
+			adaptor.checkAndHandleFpsFluctuation(streamId, 30, now);
+			adaptor.checkAndHandleFpsFluctuation(streamId, 31, now + 1000);
+			adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 2000);
+			// Should not be fluctuating
+			expect(adaptor._fpsFluctuating[streamId]).to.not.be.true;
+			expect(adaptor._fpsFluctuationCount[streamId]).to.equal(0);
+			// Should update healthy baseline
+			expect(adaptor._fpsHealthyBaseline[streamId].length).to.be.at.most(adaptor.healthyWindowSize);
+		});
+
+		it("should detect FPS fluctuation and set fluctuating state after consecutive drops", function() {
+			const streamId = 'testStream';
+			const now = Date.now();
+			// Fill healthy baseline with stable values
+			adaptor._fpsHealthyBaseline[streamId] = [30, 30, 30, 30, 30];
+			adaptor._lastHealthyAvgFps[streamId] = 30;
+			adaptor._fpsFluctuationCount[streamId] = 0;
+			// Initialize FPS history with enough values to trigger fluctuation detection
+			adaptor._fpsHistory[streamId] = [30, 30, 30, 30, 30, 30, 30, 30, 30, 30];
+			// Add values that will trigger fluctuation detection
+			adaptor.checkAndHandleFpsFluctuation(streamId, 20, now - 1000);
+			adaptor.checkAndHandleFpsFluctuation(streamId, 15, now - 500);
+			adaptor.checkAndHandleFpsFluctuation(streamId, 10, now - 250);
+			// Simulate a dramatic drop
+			adaptor.checkAndHandleFpsFluctuation(streamId, 5, now);
+			// Should now be fluctuating
+			expect(adaptor._fpsFluctuating[streamId]).to.be.true;
+			expect(adaptor._fpsFluctuationCount[streamId]).to.be.at.least(1);
+		});
+
+		it("should trigger auto-resync and reset fluctuation state when FPS stabilizes", function() {
+			// Simulate fluctuating state
+			adaptor._fpsFluctuating[streamId] = true;
+			adaptor._fpsHistory[streamId] = [20, 20, 30, 31]; // last two are stable
+			adaptor._fpsHealthyBaseline[streamId] = [30, 30];
+			adaptor._lastHealthyAvgFps[streamId] = 30;
+			adaptor._fpsFluctuationCount[streamId] = 2;
+			adaptor.playToken = "token";
+			adaptor.playRoomId = "room";
+			adaptor.playEnableTracks = [];
+			adaptor.playSubscriberId = "subid";
+			adaptor.playSubscriberCode = "subcode";
+			adaptor.playMetaData = "meta";
+			adaptor.playRole = "role";
+			// Should trigger auto-resync
+			adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 2000);
+			expect(loggerWarnStub.called).to.be.true;
+			expect(notifyEventListenersStub.calledWith("auto_resync_triggered", sinon.match.object)).to.be.true;
+			expect(stopStub.calledWith(streamId)).to.be.true;
+			// Simulate setTimeout
+			clock.tick(600);
+			expect(playStub.calledWith(
+				streamId,
+				"token",
+				"room",
+				[],
+				"subid",
+				"subcode",
+				"meta",
+				"role"
+			)).to.be.true;
+			// Should reset fluctuation state
+			expect(adaptor._fpsFluctuating[streamId]).to.be.false;
+			expect(adaptor._fpsFluctuationCount[streamId]).to.equal(0);
+		});
+
+		it("should not trigger auto-resync if cooldown not passed", function() {
+			adaptor._fpsFluctuating[streamId] = true;
+			adaptor._fpsHistory[streamId] = [20, 20, 30, 31];
+			adaptor._fpsHealthyBaseline[streamId] = [30, 30];
+			adaptor._lastHealthyAvgFps[streamId] = 30;
+			adaptor._fpsFluctuationCount[streamId] = 2;
+			adaptor._lastAutoResyncTime[streamId] = now;
+			adaptor.playToken = "token";
+			adaptor.playRoomId = "room";
+			adaptor.playEnableTracks = [];
+			adaptor.playSubscriberId = "subid";
+			adaptor.playSubscriberCode = "subcode";
+			adaptor.playMetaData = "meta";
+			adaptor.playRole = "role";
+			// Should not trigger auto-resync due to cooldown
+			adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 500);
+			expect(loggerWarnStub.called).to.be.false;
+			expect(notifyEventListenersStub.calledWith("auto_resync_triggered", sinon.match.object)).to.be.false;
+			expect(stopStub.called).to.be.false;
+			expect(playStub.called).to.be.false;
+		});
+	});
 
 });