Skip to content

Commit ce7b8df

Browse files
authored
feature(meetings): added new events for reporting inbound audio issues (#4525)
1 parent 37b6a0f commit ce7b8df

File tree

11 files changed

+474
-71
lines changed

11 files changed

+474
-71
lines changed

package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@
7777
"ws:tools": "yarn workspaces foreach --parallel --recursive --topological-dev --verbose --from '@webex/*-tools' run build:src",
7878
"package-tools": "webex-package-tools"
7979
},
80+
"resolutions": {
81+
"@webex/json-multistream": "2.3.1"
82+
},
8083
"devDependencies": {
8184
"@babel/cli": "^7.17.10",
8285
"@babel/core": "^7.17.10",

packages/@webex/media-helpers/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
"deploy:npm": "yarn npm publish"
2323
},
2424
"dependencies": {
25-
"@webex/internal-media-core": "2.18.5",
25+
"@webex/internal-media-core": "2.19.0",
2626
"@webex/ts-events": "^1.1.0",
2727
"@webex/web-media-effects": "2.27.1"
2828
},

packages/@webex/plugin-meetings/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
"dependencies": {
6363
"@webex/common": "workspace:*",
6464
"@webex/event-dictionary-ts": "^1.0.1930",
65-
"@webex/internal-media-core": "2.18.5",
65+
"@webex/internal-media-core": "2.19.0",
6666
"@webex/internal-plugin-conversation": "workspace:*",
6767
"@webex/internal-plugin-device": "workspace:*",
6868
"@webex/internal-plugin-llm": "workspace:*",

packages/@webex/plugin-meetings/src/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,7 @@ export const EVENT_TRIGGERS = {
347347
MEETING_SELF_LEFT: 'meeting:self:left',
348348
NETWORK_QUALITY: 'network:quality',
349349
MEDIA_NEGOTIATED: 'media:negotiated',
350+
MEDIA_INBOUND_AUDIO_ISSUE_DETECTED: 'media:inboundAudio:issueDetected',
350351
// the following events apply only to multistream media connections
351352
ACTIVE_SPEAKER_CHANGED: 'media:activeSpeakerChanged',
352353
REMOTE_VIDEO_SOURCE_COUNT_CHANGED: 'media:remoteVideoSourceCountChanged',

packages/@webex/plugin-meetings/src/media/properties.ts

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@ import {
99

1010
import {parse} from '@webex/ts-sdp';
1111
import {ClientEvent} from '@webex/internal-plugin-metrics';
12+
import {throttle} from 'lodash';
13+
import Metrics from '../metrics';
1214
import {MEETINGS, QUALITY_LEVELS} from '../constants';
1315
import LoggerProxy from '../common/logs/logger-proxy';
1416
import MediaConnectionAwaiter from './MediaConnectionAwaiter';
17+
import BEHAVIORAL_METRICS from '../metrics/constants';
1518

1619
export type MediaDirection = {
1720
sendAudio: boolean;
@@ -41,6 +44,8 @@ export default class MediaProperties {
4144
videoDeviceId: any;
4245
videoStream?: LocalCameraStream;
4346
namespace = MEETINGS;
47+
mediaIssueCounters: {[key: string]: number} = {};
48+
throttledSendMediaIssueMetric: ReturnType<typeof throttle>;
4449

4550
/**
4651
* @param {Object} [options] -- to auto construct
@@ -66,6 +71,15 @@ export default class MediaProperties {
6671
this.remoteQualityLevel = QUALITY_LEVELS.HIGH;
6772
this.mediaSettings = {};
6873
this.videoDeviceId = null;
74+
75+
this.throttledSendMediaIssueMetric = throttle((eventPayload) => {
76+
Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED, {
77+
...eventPayload,
78+
});
79+
Object.keys(this.mediaIssueCounters).forEach((key) => {
80+
this.mediaIssueCounters[key] = 0;
81+
});
82+
}, 1000 * 60 * 5); // at most once every 5 minutes
6983
}
7084

7185
/**
@@ -139,8 +153,14 @@ export default class MediaProperties {
139153
this.videoDeviceId = deviceId;
140154
}
141155

156+
/**
157+
* Clears the webrtcMediaConnection. This method should be called after
158+
* peer connection is closed and no longer needed.
159+
* @returns {void}
160+
*/
142161
unsetPeerConnection() {
143162
this.webrtcMediaConnection = null;
163+
this.throttledSendMediaIssueMetric.flush();
144164
}
145165

146166
/**
@@ -424,4 +444,27 @@ export default class MediaProperties {
424444
};
425445
}
426446
}
447+
448+
/**
449+
* Sends a metric about a media issue. Metrics are throttled so that we don't
450+
* send too many of them, but include a count so that we know how many issues
451+
* were detected.
452+
*
453+
* @param {string} issueType
454+
* @param {string} issueSubType
455+
* @param {string} correlationId
456+
* @returns {void}
457+
*/
458+
public sendMediaIssueMetric(issueType: string, issueSubType: string, correlationId) {
459+
const key = `${issueType}_${issueSubType}`;
460+
461+
const count = (this.mediaIssueCounters[key] || 0) + 1;
462+
463+
this.mediaIssueCounters[key] = count;
464+
465+
this.throttledSendMediaIssueMetric({
466+
correlationId,
467+
...this.mediaIssueCounters,
468+
});
469+
}
427470
}

packages/@webex/plugin-meetings/src/meeting/index.ts

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import {
2828
StatsAnalyzerEventNames,
2929
NetworkQualityEventNames,
3030
NetworkQualityMonitor,
31+
StatsMonitor,
32+
StatsMonitorEventNames,
3133
} from '@webex/internal-media-core';
3234

3335
import {
@@ -634,6 +636,7 @@ export default class Meeting extends StatelessWebexPlugin {
634636
shareStatus: string;
635637
screenShareFloorState: ScreenShareFloorStatus;
636638
statsAnalyzer: StatsAnalyzer;
639+
statsMonitor: StatsMonitor;
637640
transcription: Transcription;
638641
updateMediaConnections: (mediaConnections: any[]) => void;
639642
userDisplayHints: any;
@@ -1287,6 +1290,13 @@ export default class Meeting extends StatelessWebexPlugin {
12871290
* @memberof Meeting
12881291
*/
12891292
this.networkQualityMonitor = null;
1293+
/**
1294+
* @instance
1295+
* @type {StatsMonitor}
1296+
* @private
1297+
* @memberof Meeting
1298+
*/
1299+
this.statsMonitor = null;
12901300
/**
12911301
* Indicates network status of the webrtc media connection
12921302
* @instance
@@ -7346,10 +7356,12 @@ export default class Meeting extends StatelessWebexPlugin {
73467356
if (this.config.stats.enableStatsAnalyzer) {
73477357
// @ts-ignore - config coming from registerPlugin
73487358
this.networkQualityMonitor = new NetworkQualityMonitor(this.config.stats);
7359+
this.statsMonitor = new StatsMonitor();
73497360
this.statsAnalyzer = new StatsAnalyzer({
73507361
// @ts-ignore - config coming from registerPlugin
73517362
config: this.config.stats,
73527363
networkQualityMonitor: this.networkQualityMonitor,
7364+
statsMonitor: this.statsMonitor,
73537365
isMultistream: this.isMultistream,
73547366
});
73557367
this.shareCAEventSentStatus = {
@@ -7363,6 +7375,33 @@ export default class Meeting extends StatelessWebexPlugin {
73637375
NetworkQualityEventNames.NETWORK_QUALITY,
73647376
this.sendNetworkQualityEvent.bind(this)
73657377
);
7378+
7379+
this.statsMonitor.on(StatsMonitorEventNames.INBOUND_AUDIO_ISSUE, (data) => {
7380+
// Before forwarding any inbound audio issues to the app, make sure that we have at least one other
7381+
// participant in the meeting with unmuted audio.
7382+
// We don't check this.mediaProperties.mediaDirection here, because that's already handled in statsAnalyzer,
7383+
// so we won't get this event if we are not setup to receive any audio
7384+
const atLeastOneUnmutedOtherMember = Object.values(
7385+
this.members.membersCollection.getAll()
7386+
).find((member) => {
7387+
return !member.isSelf && !member.isPairedWithSelf && !member.isAudioMuted;
7388+
});
7389+
7390+
if (atLeastOneUnmutedOtherMember) {
7391+
this.mediaProperties.sendMediaIssueMetric(
7392+
'inbound_audio',
7393+
data.issueSubType,
7394+
this.correlationId
7395+
);
7396+
7397+
Trigger.trigger(
7398+
this,
7399+
{file: 'meeting/index', function: 'createStatsAnalyzer'},
7400+
EVENT_TRIGGERS.MEDIA_INBOUND_AUDIO_ISSUE_DETECTED,
7401+
data
7402+
);
7403+
}
7404+
});
73667405
}
73677406
}
73687407

@@ -7661,6 +7700,10 @@ export default class Meeting extends StatelessWebexPlugin {
76617700
}
76627701

76637702
this.statsAnalyzer = null;
7703+
this.networkQualityMonitor?.removeAllListeners();
7704+
this.networkQualityMonitor = null;
7705+
this.statsMonitor?.removeAllListeners();
7706+
this.statsMonitor = null;
76647707

76657708
// when media fails, we want to upload a webrtc dump to see whats going on
76667709
// this function is async, but returns once the stats have been gathered
@@ -7684,6 +7727,10 @@ export default class Meeting extends StatelessWebexPlugin {
76847727
await this.statsAnalyzer.stopAnalyzer();
76857728
}
76867729
this.statsAnalyzer = null;
7730+
this.networkQualityMonitor?.removeAllListeners();
7731+
this.networkQualityMonitor = null;
7732+
this.statsMonitor?.removeAllListeners();
7733+
this.statsMonitor = null;
76877734

76887735
this.isMultistream = false;
76897736

packages/@webex/plugin-meetings/src/metrics/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ const BEHAVIORAL_METRICS = {
8686
VERIFY_REGISTRATION_ID_SUCCESS: 'js_sdk_verify_registrationId_success',
8787
VERIFY_REGISTRATION_ID_ERROR: 'js_sdk_verify_registrationId_error',
8888
JOIN_FORBIDDEN_ERROR: 'js_sdk_join_forbidden_error',
89+
MEDIA_ISSUE_DETECTED: 'js_sdk_media_issue_detected',
8990
};
9091

9192
export {BEHAVIORAL_METRICS as default};

packages/@webex/plugin-meetings/test/unit/spec/media/properties.ts

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import * as tsSdpModule from '@webex/ts-sdp';
66
import MediaProperties from '@webex/plugin-meetings/src/media/properties';
77
import {Defer} from '@webex/common';
88
import MediaConnectionAwaiter from '../../../../src/media/MediaConnectionAwaiter';
9+
import Metrics from '../../../../src/metrics';
10+
import BEHAVIORAL_METRICS from '../../../../src/metrics/constants';
911

1012
describe('MediaProperties', () => {
1113
let mediaProperties;
@@ -389,4 +391,139 @@ describe('MediaProperties', () => {
389391
});
390392
});
391393
});
394+
395+
// issue types and subtypes used in these tests are just examples
396+
// they don't reflect real issue types/subtypes used in production
397+
describe('sendMediaIssueMetric', () => {
398+
let sendBehavioralMetricStub;
399+
let clock;
400+
401+
beforeEach(() => {
402+
clock = sinon.useFakeTimers();
403+
sendBehavioralMetricStub = sinon.stub(Metrics, 'sendBehavioralMetric');
404+
});
405+
406+
afterEach(() => {
407+
clock.restore();
408+
});
409+
410+
it('should send a behavioral metric with correct parameters', () => {
411+
const issueType = 'audio';
412+
const issueSubType = 'packet-loss';
413+
const correlationId = 'test-correlation-id-123';
414+
415+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
416+
417+
assert.calledOnce(sendBehavioralMetricStub);
418+
assert.calledWith(sendBehavioralMetricStub, BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED, {
419+
correlationId,
420+
'audio_packet-loss': 1,
421+
});
422+
});
423+
424+
it('should increment count while being throttled and reset it once metric goes out', () => {
425+
const issueType = 'video';
426+
const issueSubType = 'freeze';
427+
const correlationId = 'test-correlation-id';
428+
429+
// Call multiple times with same issue type/subtype
430+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
431+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
432+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
433+
434+
// First call should go through immediately, subsequent calls are throttled
435+
assert.calledOnce(sendBehavioralMetricStub);
436+
assert.calledWith(sendBehavioralMetricStub, BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED, {
437+
correlationId,
438+
video_freeze: 1, // Only the first call goes through due to throttling
439+
});
440+
sendBehavioralMetricStub.resetHistory();
441+
442+
assert.equal(mediaProperties.mediaIssueCounters['video_freeze'], 2); // counter should be reset after the first metric goes out, hence only 2 not 3 here
443+
444+
clock.tick(5 * 60 * 1000); // Advance time by 5 minutes to expire throttle
445+
446+
assert.calledOnceWithExactly(
447+
sendBehavioralMetricStub,
448+
BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED,
449+
{
450+
correlationId,
451+
video_freeze: 2,
452+
}
453+
);
454+
});
455+
456+
it('should track different issue types separately in counters', () => {
457+
const correlationId = 'test-correlation-id';
458+
459+
// Send different issue types
460+
mediaProperties.sendMediaIssueMetric('audio', 'packet-loss', correlationId);
461+
mediaProperties.sendMediaIssueMetric('video', 'freeze', correlationId);
462+
mediaProperties.sendMediaIssueMetric('audio', 'packet-loss', correlationId);
463+
mediaProperties.sendMediaIssueMetric('audio', 'packet-loss', correlationId);
464+
mediaProperties.sendMediaIssueMetric('audio', 'packet-loss', correlationId);
465+
mediaProperties.sendMediaIssueMetric('video', 'freeze', correlationId);
466+
467+
// First call should go through immediately, subsequent calls are throttled
468+
assert.calledOnceWithExactly(
469+
sendBehavioralMetricStub,
470+
BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED,
471+
{
472+
correlationId,
473+
'audio_packet-loss': 1,
474+
}
475+
);
476+
477+
// But the counters should be tracked separately
478+
assert.equal(mediaProperties.mediaIssueCounters['audio_packet-loss'], 3);
479+
assert.equal(mediaProperties.mediaIssueCounters['video_freeze'], 2);
480+
481+
sendBehavioralMetricStub.resetHistory();
482+
483+
clock.tick(5 * 60 * 1000); // Advance time by 5 minutes to expire throttle
484+
485+
assert.calledOnceWithExactly(
486+
sendBehavioralMetricStub,
487+
BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED,
488+
{
489+
correlationId,
490+
video_freeze: 2,
491+
'audio_packet-loss': 3,
492+
}
493+
);
494+
});
495+
496+
it('should flush throttled metrics when unsetPeerConnection is called', () => {
497+
const issueType = 'share';
498+
const issueSubType = 'connection-lost';
499+
const correlationId = 'test-correlation-id';
500+
501+
// Send metrics multiple times
502+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
503+
mediaProperties.sendMediaIssueMetric(issueType, issueSubType, correlationId);
504+
505+
// First call should go through immediately
506+
assert.calledOnceWithExactly(
507+
sendBehavioralMetricStub,
508+
BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED,
509+
{
510+
correlationId,
511+
'share_connection-lost': 1,
512+
}
513+
);
514+
sendBehavioralMetricStub.resetHistory();
515+
516+
// Call unsetPeerConnection which should flush throttled metrics
517+
mediaProperties.unsetPeerConnection();
518+
519+
assert.calledOnceWithExactly(
520+
sendBehavioralMetricStub,
521+
BEHAVIORAL_METRICS.MEDIA_ISSUE_DETECTED,
522+
{
523+
correlationId,
524+
'share_connection-lost': 1,
525+
}
526+
);
527+
});
528+
});
392529
});

0 commit comments

Comments
 (0)