diff --git a/.eslintrc b/.eslintrc index e93eaff..8b3d8cf 100644 --- a/.eslintrc +++ b/.eslintrc @@ -15,6 +15,8 @@ "no-underscore-dangle": "off", "max-len": ["error", { "code": 120 }], "import/extensions": "off", - "import/no-cycle": "off" + "import/no-cycle": "off", + "no-continue": "off", + "import/prefer-default-export": "off" } } diff --git a/README.md b/README.md index ebd8847..cba6a99 100644 --- a/README.md +++ b/README.md @@ -51,14 +51,13 @@ By default, WebRTCIssueDetector can be created with minimum of mandatory constru ```typescript import WebRTCIssueDetector, { QualityLimitationsIssueDetector, - FramesDroppedIssueDetector, - FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, OutboundNetworkIssueDetector, NetworkMediaSyncIssueDetector, AvailableOutgoingBitrateIssueDetector, UnknownVideoDecoderImplementationDetector, FrozenVideoTrackDetector, + VideoDecoderIssueDetector, } from 'webrtc-issue-detector'; const widWithDefaultConstructorArgs = new WebRTCIssueDetector(); @@ -68,14 +67,13 @@ const widWithDefaultConstructorArgs = new WebRTCIssueDetector(); const widWithCustomConstructorArgs = new WebRTCIssueDetector({ detectors: [ // you are free to change the detectors list according to your needs new QualityLimitationsIssueDetector(), - new FramesDroppedIssueDetector(), - new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), new NetworkMediaSyncIssueDetector(), new AvailableOutgoingBitrateIssueDetector(), new UnknownVideoDecoderImplementationDetector(), new FrozenVideoTrackDetector(), + new VideoDecoderIssueDetector(), ], getStatsInterval: 10_000, // set custom stats parsing interval onIssues: (payload: IssueDetectorResult) => { @@ -106,34 +104,18 @@ const exampleIssue = { } ``` -### FramesDroppedIssueDetector +### VideoDecoderIssueDetector Detects issues with decoder. ```js const exampleIssue = { type: 'cpu', reason: 'decoder-cpu-throttling', statsSample: { - deltaFramesDropped: 100, - deltaFramesReceived: 1000, - deltaFramesDecoded: 900, - framesDroppedPct: 10, + affectedStreamsPercent: 67, + throtthedStreams: [ + { ssrc: 123, allDecodeTimePerFrame: [1.2, 1.6, 1.9, 2.4, 2.9], volatility: 1.7 }, + ] }, - ssrc: 1234, -} -``` - -### FramesEncodedSentIssueDetector -Detects issues with outbound network throughput. -```js -const exampleIssue = { - type: 'network', - reason: 'outbound-network-throughput', - statsSample: { - deltaFramesSent: 900, - deltaFramesEncoded: 1000, - missedFramesPct: 10, - }, - ssrc: 1234, } ``` diff --git a/src/WebRTCIssueDetector.ts b/src/WebRTCIssueDetector.ts index 709db76..c593a80 100644 --- a/src/WebRTCIssueDetector.ts +++ b/src/WebRTCIssueDetector.ts @@ -7,6 +7,7 @@ import { IssueDetector, IssuePayload, Logger, + NetworkScores, StatsReportItem, WebRTCIssueDetectorConstructorParams, WebRTCStatsParsed, @@ -16,14 +17,13 @@ import PeriodicWebRTCStatsReporter from './parser/PeriodicWebRTCStatsReporter'; import DefaultNetworkScoresCalculator from './NetworkScoresCalculator'; import { AvailableOutgoingBitrateIssueDetector, - FramesDroppedIssueDetector, - FramesEncodedSentIssueDetector, InboundNetworkIssueDetector, NetworkMediaSyncIssueDetector, OutboundNetworkIssueDetector, QualityLimitationsIssueDetector, UnknownVideoDecoderImplementationDetector, FrozenVideoTrackDetector, + VideoDecoderIssueDetector, } from './detectors'; import { CompositeRTCStatsParser, RTCStatsParser } from './parser'; import createLogger from './utils/logger'; @@ -60,14 +60,13 @@ class WebRTCIssueDetector { this.detectors = params.detectors ?? [ new QualityLimitationsIssueDetector(), - new FramesDroppedIssueDetector(), - new FramesEncodedSentIssueDetector(), new InboundNetworkIssueDetector(), new OutboundNetworkIssueDetector(), new NetworkMediaSyncIssueDetector(), new AvailableOutgoingBitrateIssueDetector(), new UnknownVideoDecoderImplementationDetector(), new FrozenVideoTrackDetector(), + new VideoDecoderIssueDetector(), new MissingStreamDataDetector(), ]; @@ -90,11 +89,8 @@ class WebRTCIssueDetector { } this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORT_READY_EVENT, (report: StatsReportItem) => { - this.detectIssues({ - data: report.stats, - }); - - this.calculateNetworkScores(report.stats); + const networkScores = this.calculateNetworkScores(report.stats); + this.detectIssues({ data: report.stats }, networkScores); }); this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORTS_PARSED, (data: { @@ -163,16 +159,19 @@ class WebRTCIssueDetector { this.eventEmitter.emit(EventType.Issue, issues); } - private detectIssues({ data }: DetectIssuesPayload): void { - const issues = this.detectors.reduce((acc, detector) => [...acc, ...detector.detect(data)], []); + private detectIssues({ data }: DetectIssuesPayload, networkScores: NetworkScores): void { + const issues = this.detectors + .reduce((acc, detector) => [...acc, ...detector.detect(data, networkScores)], []); + if (issues.length > 0) { this.emitIssues(issues); } } - private calculateNetworkScores(data: WebRTCStatsParsed): void { + private calculateNetworkScores(data: WebRTCStatsParsed): NetworkScores { const networkScores = this.networkScoresCalculator.calculate(data); this.eventEmitter.emit(EventType.NetworkScoresUpdated, networkScores); + return networkScores; } private wrapRTCPeerConnection(): void { diff --git a/src/detectors/BaseIssueDetector.ts b/src/detectors/BaseIssueDetector.ts index 2ff4d10..cfe05a5 100644 --- a/src/detectors/BaseIssueDetector.ts +++ b/src/detectors/BaseIssueDetector.ts @@ -1,4 +1,10 @@ -import { IssueDetector, IssueDetectorResult, WebRTCStatsParsed } from '../types'; +import { + IssueDetector, + IssueDetectorResult, + NetworkScores, + WebRTCStatsParsed, + WebRTCStatsParsedWithNetworkScores, +} from '../types'; import { scheduleTask } from '../utils/tasks'; import { CLEANUP_PREV_STATS_TTL_MS, MAX_PARSED_STATS_STORAGE_SIZE } from '../utils/constants'; @@ -13,7 +19,7 @@ export interface BaseIssueDetectorParams { } abstract class BaseIssueDetector implements IssueDetector { - readonly #parsedStatsStorage: Map = new Map(); + readonly #parsedStatsStorage: Map = new Map(); readonly #statsCleanupDelayMs: number; @@ -24,11 +30,19 @@ abstract class BaseIssueDetector implements IssueDetector { this.#maxParsedStatsStorageSize = params.maxParsedStatsStorageSize ?? MAX_PARSED_STATS_STORAGE_SIZE; } - abstract performDetection(data: WebRTCStatsParsed): IssueDetectorResult; + abstract performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult; - detect(data: WebRTCStatsParsed): IssueDetectorResult { - const result = this.performDetection(data); + detect(data: WebRTCStatsParsed, networkScores?: NetworkScores): IssueDetectorResult { + const parsedStatsWithNetworkScores = { + ...data, + networkScores: { + ...networkScores, + statsSamples: networkScores?.statsSamples || {}, + }, + }; + const result = this.performDetection(parsedStatsWithNetworkScores); + this.setLastProcessedStats(data.connection.id, parsedStatsWithNetworkScores); this.performPrevStatsCleanup({ connectionId: data.connection.id, }); @@ -56,7 +70,7 @@ abstract class BaseIssueDetector implements IssueDetector { }); } - protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsed): void { + protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsedWithNetworkScores): void { if (!connectionId || parsedStats.connection.id !== connectionId) { return; } @@ -71,16 +85,16 @@ abstract class BaseIssueDetector implements IssueDetector { this.#parsedStatsStorage.set(connectionId, connectionStats); } - protected getLastProcessedStats(connectionId: string): WebRTCStatsParsed | undefined { + protected getLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores | undefined { const connectionStats = this.#parsedStatsStorage.get(connectionId); return connectionStats?.[connectionStats.length - 1]; } - protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsed[] { + protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores[] { return this.#parsedStatsStorage.get(connectionId) ?? []; } - private deleteLastProcessedStats(connectionId: string): void { + protected deleteLastProcessedStats(connectionId: string): void { this.#parsedStatsStorage.delete(connectionId); } } diff --git a/src/detectors/FramesDroppedIssueDetector.ts b/src/detectors/FramesDroppedIssueDetector.ts deleted file mode 100644 index 9512d42..0000000 --- a/src/detectors/FramesDroppedIssueDetector.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - IssueDetectorResult, - IssueReason, - IssueType, - WebRTCStatsParsed, -} from '../types'; -import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; - -interface FramesDroppedIssueDetectorParams extends BaseIssueDetectorParams { - framesDroppedThreshold?: number; -} - -class FramesDroppedIssueDetector extends BaseIssueDetector { - readonly #framesDroppedThreshold: number; - - constructor(params: FramesDroppedIssueDetectorParams = {}) { - super(params); - this.#framesDroppedThreshold = params.framesDroppedThreshold ?? 0.5; - } - - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; - } - - private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const streamsWithDroppedFrames = data.video.inbound.filter((stats) => stats.framesDropped > 0); - const issues: IssueDetectorResult = []; - const previousInboundRTPVideoStreamsStats = this.getLastProcessedStats(data.connection.id)?.video.inbound; - - if (!previousInboundRTPVideoStreamsStats) { - return issues; - } - - streamsWithDroppedFrames.forEach((streamStats) => { - const previousStreamStats = previousInboundRTPVideoStreamsStats.find((item) => item.ssrc === streamStats.ssrc); - if (!previousStreamStats) { - return; - } - - if (streamStats.framesDropped === previousStreamStats.framesDropped) { - // stream is decoded correctly - return; - } - - const deltaFramesReceived = streamStats.framesReceived - previousStreamStats.framesReceived; - const deltaFramesDecoded = streamStats.framesDecoded - previousStreamStats.framesDecoded; - const deltaFramesDropped = streamStats.framesDropped - previousStreamStats.framesDropped; - const framesDropped = deltaFramesDropped / deltaFramesReceived; - - if (deltaFramesReceived === 0 || deltaFramesDecoded === 0) { - // looks like stream is stopped, skip checking framesDropped - return; - } - - const statsSample = { - deltaFramesDropped, - deltaFramesReceived, - deltaFramesDecoded, - framesDroppedPct: Math.round(framesDropped * 100), - }; - - if (framesDropped >= this.#framesDroppedThreshold) { - // more than half of the received frames were dropped - issues.push({ - statsSample, - type: IssueType.CPU, - reason: IssueReason.DecoderCPUThrottling, - ssrc: streamStats.ssrc, - }); - } - }); - - return issues; - } -} - -export default FramesDroppedIssueDetector; diff --git a/src/detectors/FramesEncodedSentIssueDetector.ts b/src/detectors/FramesEncodedSentIssueDetector.ts deleted file mode 100644 index 87a4a9c..0000000 --- a/src/detectors/FramesEncodedSentIssueDetector.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { - IssueDetectorResult, - IssueReason, - IssueType, - WebRTCStatsParsed, -} from '../types'; -import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; - -interface FramesEncodedSentIssueDetectorParams extends BaseIssueDetectorParams { - missedFramesThreshold?: number; -} - -class FramesEncodedSentIssueDetector extends BaseIssueDetector { - readonly #missedFramesThreshold: number; - - constructor(params: FramesEncodedSentIssueDetectorParams = {}) { - super(params); - this.#missedFramesThreshold = params.missedFramesThreshold ?? 0.15; - } - - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; - } - - private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const streamsWithEncodedFrames = data.video.outbound.filter((stats) => stats.framesEncoded > 0); - const issues: IssueDetectorResult = []; - const previousOutboundRTPVideoStreamsStats = this.getLastProcessedStats(data.connection.id)?.video.outbound; - - if (!previousOutboundRTPVideoStreamsStats) { - return issues; - } - - streamsWithEncodedFrames.forEach((streamStats) => { - const previousStreamStats = previousOutboundRTPVideoStreamsStats.find((item) => item.ssrc === streamStats.ssrc); - - if (!previousStreamStats) { - return; - } - - if (streamStats.framesEncoded === previousStreamStats.framesEncoded) { - // stream is paused - return; - } - - const deltaFramesEncoded = streamStats.framesEncoded - previousStreamStats.framesEncoded; - const deltaFramesSent = streamStats.framesSent - previousStreamStats.framesSent; - const missedFrames = 1 - deltaFramesSent / deltaFramesEncoded; - - if (deltaFramesEncoded === 0) { - // stream is paused - return; - } - - if (deltaFramesEncoded === deltaFramesSent) { - // stream is ok - return; - } - - const statsSample = { - deltaFramesSent, - deltaFramesEncoded, - missedFramesPct: Math.round(missedFrames * 100), - }; - - if (missedFrames >= this.#missedFramesThreshold) { - issues.push({ - statsSample, - type: IssueType.Network, - reason: IssueReason.OutboundNetworkThroughput, - ssrc: streamStats.ssrc, - }); - } - }); - - return issues; - } -} - -export default FramesEncodedSentIssueDetector; diff --git a/src/detectors/FrozenVideoTrackDetector.ts b/src/detectors/FrozenVideoTrackDetector.ts index 7fae653..7408645 100644 --- a/src/detectors/FrozenVideoTrackDetector.ts +++ b/src/detectors/FrozenVideoTrackDetector.ts @@ -2,134 +2,112 @@ import { IssueDetectorResult, IssueReason, IssueType, - ParsedInboundVideoStreamStats, + MosQuality, WebRTCStatsParsed, + WebRTCStatsParsedWithNetworkScores, } from '../types'; +import { isSvcSpatialLayerChanged } from '../utils/video'; import BaseIssueDetector from './BaseIssueDetector'; interface FrozenVideoTrackDetectorParams { - timeoutMs?: number; - framesDroppedThreshold?: number; + avgFreezeDurationThresholdMs?: number; + frozenDurationThresholdPct?: number; + minMosQuality?: number; +} + +interface FrozenStreamStatsSample { + ssrc: number; + avgFreezeDurationMs: number; + frozenDurationPct: number; } class FrozenVideoTrackDetector extends BaseIssueDetector { - readonly #lastMarkedAt = new Map(); + readonly #avgFreezeDurationThresholdMs: number; - readonly #timeoutMs: number; + readonly #frozenDurationThresholdPct: number; - readonly #framesDroppedThreshold: number; + readonly #minMosQuality: MosQuality; constructor(params: FrozenVideoTrackDetectorParams = {}) { super(); - this.#timeoutMs = params.timeoutMs ?? 10_000; - this.#framesDroppedThreshold = params.framesDroppedThreshold ?? 0.5; + this.#avgFreezeDurationThresholdMs = params.avgFreezeDurationThresholdMs ?? 1_000; + this.#frozenDurationThresholdPct = params.frozenDurationThresholdPct ?? 30; + this.#minMosQuality = params.minMosQuality ?? MosQuality.BAD; } - performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { + const inboundScore = data.networkScores.inbound; + if (inboundScore !== undefined && inboundScore <= this.#minMosQuality) { + // do not execute detection on stats based on poor network quality + // to avoid false positives + return []; + } + + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const previousStats = this.getLastProcessedStats(connectionId); const issues: IssueDetectorResult = []; - - if (!previousStats) { - return issues; + const allLastProcessedStats = this.getAllLastProcessedStats(data.connection.id); + if (allLastProcessedStats.length === 0) { + return []; } - const { video: { inbound: newInbound } } = data; - const { video: { inbound: prevInbound } } = previousStats; - - const mapByTrackId = (items: ParsedInboundVideoStreamStats[]) => new Map( - items.map((item) => [item.track.trackIdentifier, item] as const), - ); - - const newInboundByTrackId = mapByTrackId(newInbound); - const prevInboundByTrackId = mapByTrackId(prevInbound); - const unvisitedTrackIds = new Set(this.#lastMarkedAt.keys()); - - Array.from(newInboundByTrackId.entries()).forEach(([trackId, newInboundItem]) => { - unvisitedTrackIds.delete(trackId); - - const prevInboundItem = prevInboundByTrackId.get(trackId); - if (!prevInboundItem) { - return; - } - - const deltaFramesReceived = newInboundItem.framesReceived - prevInboundItem.framesReceived; - const deltaFramesDropped = newInboundItem.framesDropped - prevInboundItem.framesDropped; - const deltaFramesDecoded = newInboundItem.framesDecoded - prevInboundItem.framesDecoded; - const ratioFramesDropped = deltaFramesDropped / deltaFramesReceived; - - if (deltaFramesReceived === 0) { - return; - } - - // We skip it when ratio is too low because it should be handled by FramesDroppedIssueDetector - if (ratioFramesDropped >= this.#framesDroppedThreshold) { - return; - } - - // It seems that track is alive and we can remove mark if it was marked - if (deltaFramesDecoded > 0) { - this.removeMarkIssue(trackId); - return; - } - - const hasIssue = this.markIssue(trackId); - - if (!hasIssue) { - return; - } - - const statsSample = { - framesReceived: newInboundItem.framesReceived, - framesDropped: newInboundItem.framesDropped, - framesDecoded: newInboundItem.framesDecoded, - deltaFramesReceived, - deltaFramesDropped, - deltaFramesDecoded, - }; - + const frozenStreams = data.video.inbound + .map((videoStream): FrozenStreamStatsSample | undefined => { + const prevStat = allLastProcessedStats[allLastProcessedStats.length - 1] + .video.inbound.find((stream) => stream.ssrc === videoStream.ssrc); + + if (!prevStat) { + return undefined; + } + + const isSpatialLayerChanged = isSvcSpatialLayerChanged(videoStream.ssrc, [ + allLastProcessedStats[allLastProcessedStats.length - 1], + data, + ]); + + if (isSpatialLayerChanged) { + return undefined; + } + + const deltaFreezeCount = videoStream.freezeCount - (prevStat.freezeCount ?? 0); + const deltaFreezesTimeMs = (videoStream.totalFreezesDuration - (prevStat.totalFreezesDuration ?? 0)) * 1000; + const avgFreezeDurationMs = deltaFreezeCount > 0 ? deltaFreezesTimeMs / deltaFreezeCount : 0; + + const statsTimeDiff = videoStream.timestamp - prevStat.timestamp; + const frozenDurationPct = (deltaFreezesTimeMs / statsTimeDiff) * 100; + if (frozenDurationPct > this.#frozenDurationThresholdPct) { + return { + ssrc: videoStream.ssrc, + avgFreezeDurationMs, + frozenDurationPct, + }; + } + + if (avgFreezeDurationMs > this.#avgFreezeDurationThresholdMs) { + return { + ssrc: videoStream.ssrc, + avgFreezeDurationMs, + frozenDurationPct, + }; + } + + return undefined; + }) + .filter((stream) => stream !== undefined) as FrozenStreamStatsSample[]; + + if (frozenStreams.length > 0) { issues.push({ - statsSample, type: IssueType.Stream, reason: IssueReason.FrozenVideoTrack, - trackIdentifier: trackId, + statsSample: { + ssrcs: frozenStreams.map((stream) => stream.ssrc), + }, }); - }); - - // just clear unvisited tracks from memory - unvisitedTrackIds.forEach((trackId) => { - this.removeMarkIssue(trackId); - }); - - return issues; - } - - private markIssue(trackId: string): boolean { - const now = Date.now(); - - const lastMarkedAt = this.#lastMarkedAt.get(trackId); - - if (!lastMarkedAt) { - this.#lastMarkedAt.set(trackId, now); - return false; - } - - if (now - lastMarkedAt < this.#timeoutMs) { - return false; } - return true; - } - - private removeMarkIssue(trackId: string): void { - this.#lastMarkedAt.delete(trackId); + return issues; } } diff --git a/src/detectors/InboundNetworkIssueDetector.ts b/src/detectors/InboundNetworkIssueDetector.ts index f1736a3..bbb4ed2 100644 --- a/src/detectors/InboundNetworkIssueDetector.ts +++ b/src/detectors/InboundNetworkIssueDetector.ts @@ -31,10 +31,7 @@ class InboundNetworkIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/MissingStreamDataDetector.ts b/src/detectors/MissingStreamDataDetector.ts index a0b5ef3..e61f035 100644 --- a/src/detectors/MissingStreamDataDetector.ts +++ b/src/detectors/MissingStreamDataDetector.ts @@ -27,10 +27,7 @@ export default class MissingStreamDataDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/NetworkMediaSyncIssueDetector.ts b/src/detectors/NetworkMediaSyncIssueDetector.ts index 8bacd94..7fa36c9 100644 --- a/src/detectors/NetworkMediaSyncIssueDetector.ts +++ b/src/detectors/NetworkMediaSyncIssueDetector.ts @@ -19,10 +19,7 @@ class NetworkMediaSyncIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/OutboundNetworkIssueDetector.ts b/src/detectors/OutboundNetworkIssueDetector.ts index 95a637a..399ca55 100644 --- a/src/detectors/OutboundNetworkIssueDetector.ts +++ b/src/detectors/OutboundNetworkIssueDetector.ts @@ -23,10 +23,7 @@ class OutboundNetworkIssueDetector extends BaseIssueDetector { } performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/QualityLimitationsIssueDetector.ts b/src/detectors/QualityLimitationsIssueDetector.ts index c78e9f5..8a25a80 100644 --- a/src/detectors/QualityLimitationsIssueDetector.ts +++ b/src/detectors/QualityLimitationsIssueDetector.ts @@ -8,10 +8,7 @@ import BaseIssueDetector from './BaseIssueDetector'; class QualityLimitationsIssueDetector extends BaseIssueDetector { performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } private processData(data: WebRTCStatsParsed): IssueDetectorResult { diff --git a/src/detectors/UnknownVideoDecoderImplementationDetector.ts b/src/detectors/UnknownVideoDecoderImplementationDetector.ts index 139dd39..18793e5 100644 --- a/src/detectors/UnknownVideoDecoderImplementationDetector.ts +++ b/src/detectors/UnknownVideoDecoderImplementationDetector.ts @@ -14,10 +14,7 @@ class UnknownVideoDecoderImplementationDetector extends BaseIssueDetector { } = {}; performDetection(data: WebRTCStatsParsed): IssueDetectorResult { - const { connection: { id: connectionId } } = data; - const issues = this.processData(data); - this.setLastProcessedStats(connectionId, data); - return issues; + return this.processData(data); } protected performPrevStatsCleanup(payload: PrevStatsCleanupPayload) { diff --git a/src/detectors/VideoDecoderIssueDetector.ts b/src/detectors/VideoDecoderIssueDetector.ts new file mode 100644 index 0000000..e879daa --- /dev/null +++ b/src/detectors/VideoDecoderIssueDetector.ts @@ -0,0 +1,118 @@ +import { calculateVolatility } from '../helpers/calc'; +import { + IssueDetectorResult, + IssueReason, + IssueType, + MosQuality, + WebRTCStatsParsedWithNetworkScores, +} from '../types'; +import { isSvcSpatialLayerChanged } from '../utils/video'; +import BaseIssueDetector, { BaseIssueDetectorParams } from './BaseIssueDetector'; + +interface VideoDecoderIssueDetectorParams extends BaseIssueDetectorParams { + volatilityThreshold?: number; + affectedStreamsPercentThreshold?: number; + minMosQuality?: number; +} + +class VideoDecoderIssueDetector extends BaseIssueDetector { + readonly #volatilityThreshold: number; + + readonly #affectedStreamsPercentThreshold: number; + + readonly #minMosQuality: MosQuality; + + constructor(params: VideoDecoderIssueDetectorParams = {}) { + super(params); + this.#volatilityThreshold = params.volatilityThreshold ?? 8; + this.#affectedStreamsPercentThreshold = params.affectedStreamsPercentThreshold ?? 30; + this.#minMosQuality = params.minMosQuality ?? MosQuality.BAD; + } + + performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { + const allHistoricalStats = [ + ...this.getAllLastProcessedStats(data.connection.id), + data, + ]; + + const isBadNetworkHappened = allHistoricalStats + .find((stat) => stat.networkScores.inbound !== undefined && stat.networkScores.inbound <= this.#minMosQuality); + + if (isBadNetworkHappened) { + // do not execute detection on historical stats based on bad network quality + // to avoid false positives + return []; + } + + return this.processData(data); + } + + private processData(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult { + const issues: IssueDetectorResult = []; + + const allProcessedStats = [ + ...this.getAllLastProcessedStats(data.connection.id), + data, + ]; + + const throtthedStreams = data.video.inbound + .map((incomeVideoStream): { ssrc: number, allFps: number[], volatility: number } | undefined => { + // At least 5 elements needed to have enough representation + if (allProcessedStats.length < 5) { + return undefined; + } + + const isSpatialLayerChanged = isSvcSpatialLayerChanged(incomeVideoStream.ssrc, allProcessedStats); + if (isSpatialLayerChanged) { + return undefined; + } + + const allFps: number[] = []; + for (let i = 0; i < allProcessedStats.length - 1; i += 1) { + const videoStreamStats = allProcessedStats[i].video.inbound.find( + (stream) => stream.ssrc === incomeVideoStream.ssrc, + ); + + if (videoStreamStats?.framesPerSecond !== undefined) { + allFps.push(videoStreamStats.framesPerSecond); + } + } + + if (allFps.length === 0) { + return undefined; + } + + const volatility = calculateVolatility(allFps); + + if (volatility > this.#volatilityThreshold) { + return { ssrc: incomeVideoStream.ssrc, allFps, volatility }; + } + + return undefined; + }) + .filter((throttledVideoStream) => Boolean(throttledVideoStream)); + + if (throtthedStreams.length === 0) { + return issues; + } + + const affectedStreamsPercent = throtthedStreams.length / (data.video.inbound.length / 100); + if (affectedStreamsPercent > this.#affectedStreamsPercentThreshold) { + issues.push({ + type: IssueType.CPU, + reason: IssueReason.DecoderCPUThrottling, + statsSample: { + affectedStreamsPercent, + throtthedStreams, + }, + }); + + // clear all processed stats for this connection to avoid duplicate issues + this.deleteLastProcessedStats(data.connection.id); + } + + return issues; + } +} + +export default VideoDecoderIssueDetector; diff --git a/src/detectors/index.ts b/src/detectors/index.ts index d797b85..2c7e266 100644 --- a/src/detectors/index.ts +++ b/src/detectors/index.ts @@ -1,10 +1,9 @@ export { default as BaseIssueDetector } from './BaseIssueDetector'; export { default as AvailableOutgoingBitrateIssueDetector } from './AvailableOutgoingBitrateIssueDetector'; -export { default as FramesDroppedIssueDetector } from './FramesDroppedIssueDetector'; -export { default as FramesEncodedSentIssueDetector } from './FramesEncodedSentIssueDetector'; export { default as InboundNetworkIssueDetector } from './InboundNetworkIssueDetector'; export { default as NetworkMediaSyncIssueDetector } from './NetworkMediaSyncIssueDetector'; export { default as OutboundNetworkIssueDetector } from './OutboundNetworkIssueDetector'; export { default as QualityLimitationsIssueDetector } from './QualityLimitationsIssueDetector'; export { default as UnknownVideoDecoderImplementationDetector } from './UnknownVideoDecoderImplementationDetector'; export { default as FrozenVideoTrackDetector } from './FrozenVideoTrackDetector'; +export { default as VideoDecoderIssueDetector } from './VideoDecoderIssueDetector'; diff --git a/src/helpers/calc.ts b/src/helpers/calc.ts new file mode 100644 index 0000000..29db3ed --- /dev/null +++ b/src/helpers/calc.ts @@ -0,0 +1,11 @@ +export const calculateMean = (values: number[]) => values.reduce((acc, val) => acc + val, 0) / values.length; + +export const calculateVolatility = (values: number[]) => { + if (values.length === 0) { + throw new Error('Cannot calculate volatility for empty array'); + } + + const mean = calculateMean(values); + const meanAbsoluteDeviationFps = values.reduce((acc, val) => acc + Math.abs(val - mean), 0) / values.length; + return (meanAbsoluteDeviationFps * 100) / mean; +}; diff --git a/src/types.ts b/src/types.ts index d56b8d0..84a1c6c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,7 +11,7 @@ export interface WIDWindow { export type IssueDetectorResult = IssuePayload[]; export interface IssueDetector { - detect(data: WebRTCStatsParsed): IssueDetectorResult; + detect(data: WebRTCStatsParsed, networkScores?: NetworkScores): IssueDetectorResult; } export interface INetworkScoresCalculator { @@ -253,6 +253,8 @@ export type ParsedInboundVideoStreamStats = { totalDecodeTime: number, totalInterFrameDelay: number, totalSquaredInterFrameDelay: number, + freezeCount: number, + totalFreezesDuration: number, track: { detached: boolean, ended: boolean, @@ -436,6 +438,16 @@ export interface Logger { error: (msg: any, ...meta: any[]) => void; } +export enum MosQuality { + BAD = 2.1, + POOR = 2.6, + FAIR = 3.1, + GOOD = 3.8, + EXCELLENT = 4.3, +} + +export type WebRTCStatsParsedWithNetworkScores = WebRTCStatsParsed & { networkScores: NetworkScores }; + type CommonKeys = Extract; type CommonFields = { diff --git a/src/utils/video.ts b/src/utils/video.ts new file mode 100644 index 0000000..0dbb5b1 --- /dev/null +++ b/src/utils/video.ts @@ -0,0 +1,25 @@ +import { WebRTCStatsParsed } from '../types'; + +export const isSvcSpatialLayerChanged = (ssrc: number, allProcessedStats: WebRTCStatsParsed[]): boolean => { + for (let i = 1; i < allProcessedStats.length; i += 1) { + const videoStreamStats = allProcessedStats[i].video.inbound.find( + (stream) => stream.ssrc === ssrc, + ); + + if (!videoStreamStats) { + continue; + } + + const prevVideoStreamStats = allProcessedStats[i - 1].video.inbound.find( + (stream) => stream.ssrc === ssrc, + ); + + const widthChanged = videoStreamStats.frameWidth !== prevVideoStreamStats?.frameWidth; + const heightChanged = videoStreamStats.frameHeight !== prevVideoStreamStats?.frameHeight; + if (widthChanged || heightChanged) { + return true; + } + } + + return false; +};