Skip to content

Commit 8d1053c

Browse files
robwalchrobwalch
authored
Timestamp offset stability fix for muxed audiovideo mp4 (#7436)
* Fix resetting of initPTS when muxed "audiovideo" sample start times are inconsistent Fixes #7431 * Continue loading parts even with gaps and warn when frag selects fallback likely due to a gap
1 parent 2fb519b commit 8d1053c

16 files changed

+149
-81
lines changed

‎api-extractor/report/hls.js.api.md

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ export class AudioStreamController extends BaseStreamController implements Netwo
197197
// (undocumented)
198198
protected onHandlerDestroying(): void;
199199
// (undocumented)
200-
onInitPtsFound(event: Events.INIT_PTS_FOUND, { frag, id, initPTS, timescale }: InitPTSFoundData): void;
200+
onInitPtsFound(event: Events.INIT_PTS_FOUND, { frag, id, initPTS, timescale, trackId }: InitPTSFoundData): void;
201201
// (undocumented)
202202
protected onManifestLoading(): void;
203203
// (undocumented)
@@ -458,7 +458,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
458458
// (undocumented)
459459
get inFlightFrag(): InFlightData;
460460
// (undocumented)
461-
protected initPTS: RationalTimestamp[];
461+
protected initPTS: TimestampOffset[];
462462
// (undocumented)
463463
protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean;
464464
// (undocumented)
@@ -2584,6 +2584,8 @@ export interface InitPTSFoundData {
25842584
initPTS: number;
25852585
// (undocumented)
25862586
timescale: number;
2587+
// (undocumented)
2588+
trackId: number;
25872589
}
25882590

25892591
// Warning: (ae-missing-release-tag) "InitSegmentData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -4796,6 +4798,13 @@ export enum TimelineOccupancy {
47964798
Range = 1
47974799
}
47984800

4801+
// Warning: (ae-missing-release-tag) "TimestampOffset" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
4802+
//
4803+
// @public (undocumented)
4804+
export type TimestampOffset = RationalTimestamp & {
4805+
trackId: number;
4806+
};
4807+
47994808
// Warning: (ae-missing-release-tag) "Track" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
48004809
//
48014810
// @public (undocumented)
@@ -4866,7 +4875,7 @@ export class TransmuxerInterface {
48664875
// (undocumented)
48674876
flush(chunkMeta: ChunkMetadata): void;
48684877
// (undocumented)
4869-
push(data: ArrayBuffer, initSegmentData: Uint8Array | undefined, audioCodec: string | undefined, videoCodec: string | undefined, frag: MediaFragment, part: Part | null, duration: number, accurateTimeOffset: boolean, chunkMeta: ChunkMetadata, defaultInitPTS?: RationalTimestamp): void;
4878+
push(data: ArrayBuffer, initSegmentData: Uint8Array | undefined, audioCodec: string | undefined, videoCodec: string | undefined, frag: MediaFragment, part: Part | null, duration: number, accurateTimeOffset: boolean, chunkMeta: ChunkMetadata, defaultInitPTS?: TimestampOffset): void;
48704879
// (undocumented)
48714880
reset(): void;
48724881
}

‎src/controller/audio-stream-controller.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,16 +142,16 @@ class AudioStreamController
142142
// INIT_PTS_FOUND is triggered when the video track parsed in the stream-controller has a new PTS value
143143
onInitPtsFound(
144144
event: Events.INIT_PTS_FOUND,
145-
{ frag, id, initPTS, timescale }: InitPTSFoundData,
145+
{ frag, id, initPTS, timescale, trackId }: InitPTSFoundData,
146146
) {
147147
// Always update the new INIT PTS
148148
// Can change due level switch
149149
if (id === PlaylistLevelType.MAIN) {
150150
const cc = frag.cc;
151151
const inFlightFrag = this.fragCurrent;
152-
this.initPTS[cc] = { baseTime: initPTS, timescale };
152+
this.initPTS[cc] = { baseTime: initPTS, timescale, trackId };
153153
this.log(
154-
`InitPTS for cc: ${cc} found from main: ${initPTS}/${timescale}`,
154+
`InitPTS for cc: ${cc} found from main: ${initPTS / timescale} (${initPTS}/${timescale}) trackId: ${trackId}`,
155155
);
156156
this.mainAnchor = frag;
157157
// If we are waiting, tick immediately to unblock audio fragment transmuxing

‎src/controller/base-stream-controller.ts

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ import type {
5959
import type { Level } from '../types/level';
6060
import type { RemuxedTrack } from '../types/remuxer';
6161
import type { Bufferable, BufferInfo } from '../utils/buffer-helper';
62-
import type { RationalTimestamp } from '../utils/timescale-conversion';
62+
import type { TimestampOffset } from '../utils/timescale-conversion';
6363

6464
type ResolveFragLoaded = (FragLoadedEndData) => void;
6565
type RejectFragLoaded = (LoadError) => void;
@@ -111,7 +111,7 @@ export default class BaseStreamController
111111
protected levelLastLoaded: Level | null = null;
112112
protected startFragRequested: boolean = false;
113113
protected decrypter: Decrypter;
114-
protected initPTS: RationalTimestamp[] = [];
114+
protected initPTS: TimestampOffset[] = [];
115115
protected buffering: boolean = true;
116116
protected loadingParts: boolean = false;
117117
private loopSn?: string | number;
@@ -912,7 +912,7 @@ export default class BaseStreamController
912912
this.log(
913913
`LL-Part loading OFF after next part miss @${targetBufferTime.toFixed(
914914
2,
915-
)}`,
915+
)} Check buffer at sn: ${frag.sn} loaded parts: ${details.partList?.filter((p) => p.loaded).map((p) => `[${p.start}-${p.end}]`)}`,
916916
);
917917
this.loadingParts = false;
918918
} else if (!frag.url) {
@@ -1496,9 +1496,14 @@ export default class BaseStreamController
14961496
if (loaded) {
14971497
nextPart = -1;
14981498
} else if (
1499-
(contiguous || part.independent || independentAttrOmitted) &&
1500-
part.fragment === frag
1499+
contiguous ||
1500+
((part.independent || independentAttrOmitted) && part.fragment === frag)
15011501
) {
1502+
if (part.fragment !== frag) {
1503+
this.warn(
1504+
`Need buffer at ${targetBufferTime} but next unloaded part starts at ${part.start}`,
1505+
);
1506+
}
15021507
nextPart = i;
15031508
}
15041509
contiguous = loaded;
@@ -1510,8 +1515,17 @@ export default class BaseStreamController
15101515
partList: Part[],
15111516
targetBufferTime: number,
15121517
): boolean {
1513-
const lastPart = partList[partList.length - 1];
1514-
return lastPart && targetBufferTime > lastPart.start && lastPart.loaded;
1518+
let part: Part;
1519+
for (let i = partList.length; i--; ) {
1520+
part = partList[i];
1521+
if (!part.loaded) {
1522+
return false;
1523+
}
1524+
if (targetBufferTime > part.start) {
1525+
return true;
1526+
}
1527+
}
1528+
return false;
15151529
}
15161530

15171531
/*

‎src/controller/stream-controller.ts

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,7 +1248,6 @@ export default class StreamController
12481248
});
12491249
}
12501250

1251-
// This would be nice if Number.isFinite acted as a typeguard, but it doesn't. See: https://github.com/Microsoft/TypeScript/issues/10038
12521251
const baseTime = initSegment.initPTS as number;
12531252
const timescale = initSegment.timescale as number;
12541253
const initPTS = this.initPTS[frag.cc];
@@ -1258,12 +1257,18 @@ export default class StreamController
12581257
initPTS.baseTime !== baseTime ||
12591258
initPTS.timescale !== timescale)
12601259
) {
1261-
this.initPTS[frag.cc] = { baseTime, timescale };
1260+
const trackId = initSegment.trackId as number;
1261+
this.initPTS[frag.cc] = {
1262+
baseTime,
1263+
timescale,
1264+
trackId,
1265+
};
12621266
hls.trigger(Events.INIT_PTS_FOUND, {
12631267
frag,
12641268
id,
12651269
initPTS: baseTime,
12661270
timescale,
1271+
trackId,
12671272
});
12681273
}
12691274
}

‎src/controller/timeline-controller.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import type { MediaPlaylist } from '../types/media-playlist';
3636
import type { VTTCCs } from '../types/vtt';
3737
import type { CaptionScreen } from '../utils/cea-608-parser';
3838
import type { CuesInterface } from '../utils/cues';
39-
import type { RationalTimestamp } from '../utils/timescale-conversion';
39+
import type { TimestampOffset } from '../utils/timescale-conversion';
4040

4141
type TrackProperties = {
4242
label: string;
@@ -61,7 +61,7 @@ export class TimelineController implements ComponentAPI {
6161
private Cues: CuesInterface;
6262
private textTracks: Array<TextTrack> = [];
6363
private tracks: Array<MediaPlaylist> = [];
64-
private initPTS: RationalTimestamp[] = [];
64+
private initPTS: TimestampOffset[] = [];
6565
private unparsedVttFrags: Array<FragLoadedData | FragDecryptedData> = [];
6666
private captionsTracks: Record<string, TextTrack> = {};
6767
private nonNativeCaptionsTracks: Record<string, NonNativeCaptionsTrack> = {};
@@ -191,11 +191,11 @@ export class TimelineController implements ComponentAPI {
191191
// Triggered when an initial PTS is found; used for synchronisation of WebVTT.
192192
private onInitPtsFound(
193193
event: Events.INIT_PTS_FOUND,
194-
{ frag, id, initPTS, timescale }: InitPTSFoundData,
194+
{ frag, id, initPTS, timescale, trackId }: InitPTSFoundData,
195195
) {
196196
const { unparsedVttFrags } = this;
197197
if (id === PlaylistLevelType.MAIN) {
198-
this.initPTS[frag.cc] = { baseTime: initPTS, timescale };
198+
this.initPTS[frag.cc] = { baseTime: initPTS, timescale, trackId };
199199
}
200200

201201
// Due to asynchronous processing, initial PTS may arrive later than the first VTT fragments are loaded.

‎src/demux/audio/base-audio-demuxer.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,18 @@ import {
1414
} from '../../types/demuxer';
1515
import { appendUint8Array } from '../../utils/mp4-tools';
1616
import { dummyTrack } from '../dummy-demuxed-track';
17-
import type { RationalTimestamp } from '../../utils/timescale-conversion';
17+
import type {
18+
RationalTimestamp,
19+
TimestampOffset,
20+
} from '../../utils/timescale-conversion';
1821

1922
class BaseAudioDemuxer implements Demuxer {
2023
protected _audioTrack?: DemuxedAudioTrack;
2124
protected _id3Track?: DemuxedMetadataTrack;
2225
protected frameIndex: number = 0;
2326
protected cachedData: Uint8Array | null = null;
2427
protected basePTS: number | null = null;
25-
protected initPTS: RationalTimestamp | null = null;
28+
protected initPTS: TimestampOffset | null = null;
2629
protected lastPTS: number | null = null;
2730

2831
resetInitSegment(
@@ -42,7 +45,7 @@ class BaseAudioDemuxer implements Demuxer {
4245
};
4346
}
4447

45-
resetTimeStamp(deaultTimestamp: RationalTimestamp | null) {
48+
resetTimeStamp(deaultTimestamp: TimestampOffset | null) {
4649
this.initPTS = deaultTimestamp;
4750
this.resetContiguity();
4851
}

‎src/demux/transmuxer-interface.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import type Hls from '../hls';
2121
import type { MediaFragment, Part } from '../loader/fragment';
2222
import type { ErrorData, FragDecryptedData } from '../types/events';
2323
import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
24-
import type { RationalTimestamp } from '../utils/timescale-conversion';
24+
import type { TimestampOffset } from '../utils/timescale-conversion';
2525

2626
let transmuxerInstanceCount: number = 0;
2727

@@ -193,7 +193,7 @@ export default class TransmuxerInterface {
193193
duration: number,
194194
accurateTimeOffset: boolean,
195195
chunkMeta: ChunkMetadata,
196-
defaultInitPTS?: RationalTimestamp,
196+
defaultInitPTS?: TimestampOffset,
197197
) {
198198
chunkMeta.transmuxing.start = self.performance.now();
199199
const { instanceNo, transmuxer } = this;

‎src/demux/transmuxer.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import type { Remuxer } from '../types/remuxer';
2121
import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
2222
import type { TypeSupported } from '../utils/codecs';
2323
import type { ILogger } from '../utils/logger';
24-
import type { RationalTimestamp } from '../utils/timescale-conversion';
24+
import type { TimestampOffset } from '../utils/timescale-conversion';
2525

2626
let now: () => number;
2727
// performance.now() not available on WebWorker, at least on Safari Desktop
@@ -312,7 +312,7 @@ export default class Transmuxer {
312312
chunkMeta.transmuxing.executeEnd = now();
313313
}
314314

315-
resetInitialTimestamp(defaultInitPts: RationalTimestamp | null) {
315+
resetInitialTimestamp(defaultInitPts: TimestampOffset | null) {
316316
const { demuxer, remuxer } = this;
317317
if (!demuxer || !remuxer) {
318318
return;
@@ -517,14 +517,14 @@ export class TransmuxConfig {
517517
public videoCodec?: string;
518518
public initSegmentData?: Uint8Array;
519519
public duration: number;
520-
public defaultInitPts: RationalTimestamp | null;
520+
public defaultInitPts: TimestampOffset | null;
521521

522522
constructor(
523523
audioCodec: string | undefined,
524524
videoCodec: string | undefined,
525525
initSegmentData: Uint8Array | undefined,
526526
duration: number,
527-
defaultInitPts?: RationalTimestamp,
527+
defaultInitPts?: TimestampOffset,
528528
) {
529529
this.audioCodec = audioCodec;
530530
this.videoCodec = videoCodec;

‎src/hls.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1533,4 +1533,7 @@ export type {
15331533
KeySystems,
15341534
KeySystemFormats,
15351535
} from './utils/mediakeys-helper';
1536-
export type { RationalTimestamp } from './utils/timescale-conversion';
1536+
export type {
1537+
RationalTimestamp,
1538+
TimestampOffset,
1539+
} from './utils/timescale-conversion';

‎src/remux/mp4-remuxer.ts

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,10 @@ import type {
2727
} from '../types/remuxer';
2828
import type { TrackSet } from '../types/track';
2929
import type { TypeSupported } from '../utils/codecs';
30-
import type { RationalTimestamp } from '../utils/timescale-conversion';
30+
import type {
31+
RationalTimestamp,
32+
TimestampOffset,
33+
} from '../utils/timescale-conversion';
3134

3235
const MAX_SILENT_FRAME_DURATION = 10 * 1000; // 10 seconds
3336
const AAC_SAMPLES_PER_FRAME = 1024;
@@ -62,8 +65,8 @@ export default class MP4Remuxer extends Logger implements Remuxer {
6265
private readonly config: HlsConfig;
6366
private readonly typeSupported: TypeSupported;
6467
private ISGenerated: boolean = false;
65-
private _initPTS: RationalTimestamp | null = null;
66-
private _initDTS: RationalTimestamp | null = null;
68+
private _initPTS: TimestampOffset | null = null;
69+
private _initDTS: TimestampOffset | null = null;
6770
private nextVideoTs: number | null = null;
6871
private nextAudioTs: number | null = null;
6972
private videoSampleDuration: number | null = null;
@@ -103,7 +106,7 @@ export default class MP4Remuxer extends Logger implements Remuxer {
103106
this.config = this.videoTrackConfig = this._initPTS = this._initDTS = null;
104107
}
105108

106-
resetTimeStamp(defaultTimeStamp: RationalTimestamp | null) {
109+
resetTimeStamp(defaultTimeStamp: TimestampOffset | null) {
107110
this.log('initPTS & initDTS reset');
108111
this._initPTS = this._initDTS = defaultTimeStamp;
109112
}
@@ -347,7 +350,7 @@ export default class MP4Remuxer extends Logger implements Remuxer {
347350
let initPTS: number | undefined;
348351
let initDTS: number | undefined;
349352
let timescale: number | undefined;
350-
let trackId: number | undefined;
353+
let trackId: number = -1;
351354

352355
if (computePTSDTS) {
353356
initPTS = initDTS = Infinity;
@@ -439,13 +442,23 @@ export default class MP4Remuxer extends Logger implements Remuxer {
439442
if (Object.keys(tracks).length) {
440443
this.ISGenerated = true;
441444
if (computePTSDTS) {
445+
if (_initPTS) {
446+
this.warn(
447+
`Timestamps at playlist time: ${accurateTimeOffset ? '' : '~'}${timeOffset} ${initPTS! / timescale!} != initPTS: ${_initPTS.baseTime / _initPTS.timescale} (${_initPTS.baseTime}/${_initPTS.timescale}) trackId: ${_initPTS.trackId}`,
448+
);
449+
}
450+
this.log(
451+
`Found initPTS at playlist time: ${timeOffset} offset: ${initPTS! / timescale!} (${initPTS}/${timescale}) trackId: ${trackId}`,
452+
);
442453
this._initPTS = {
443454
baseTime: initPTS as number,
444455
timescale: timescale as number,
456+
trackId: trackId as number,
445457
};
446458
this._initDTS = {
447459
baseTime: initDTS as number,
448460
timescale: timescale as number,
461+
trackId: trackId as number,
449462
};
450463
} else {
451464
initPTS = timescale = undefined;
@@ -1134,8 +1147,8 @@ function findKeyframeIndex(samples: Array<VideoSample>): number {
11341147
export function flushTextTrackMetadataCueSamples(
11351148
track: DemuxedMetadataTrack,
11361149
timeOffset: number,
1137-
initPTS: RationalTimestamp,
1138-
initDTS: RationalTimestamp,
1150+
initPTS: TimestampOffset,
1151+
initDTS: TimestampOffset,
11391152
): RemuxedMetadata | undefined {
11401153
const length = track.samples.length;
11411154
if (!length) {

0 commit comments

Comments
 (0)