ngx-live: fix audio ends before video issue (#208)

* ngx-live: fix audio ends before video issue

the segmenter may try to cut a segment at the audio end timestamp, there
is no video key frame at this timestamp - the video segments come out
empty, and some of the audio frames are discarded. then it tries to
create another segment, but the segments of all tracks come out empty,
and the channel is failed.
the fix is to avoid averaging the min/max split pts when the span is too
large. in this case, the target pts will be the audio end pts, the video
segments will come out empty, and the trailing audio will be discarded.

* update readme

* update UT

nginx-live-module/README.md

@@ -1179,6 +1179,15 @@ tweaks the timestamp of the existing candidate instead of adding a new value to
 Sets a margin around the minimum span for segment pts candidates.
 When choosing the end pts for a segment, candidates whose span is greater than the minimum span by more than the configured margin are disqualified.
 
+#### segmenter_max_span_average
+* **syntax**: `segmenter_max_span_average msec;`
+* **default**: `500ms`
+* **context**: `live`, `preset`
+
+Sets a maximum value for averaging the min/max segment split pts values.
+When choosing the end pts for a segment, if the difference between the min/max split pts of the different tracks is lower than this value, the average of the min/max is used.
+Otherwise, the original candidate pts is used.
+
 #### segmenter_ready_threshold
 * **syntax**: `segmenter_ready_threshold percent;`
 * **default**: `150`

nginx-live-module/src/ngx_live_segmenter.c

@@ -59,6 +59,7 @@ typedef struct {
 
     ngx_msec_t                        candidate_margin;
     ngx_msec_t                        keyframe_alignment_margin;
+    ngx_msec_t                        max_span_average;
 
     ngx_uint_t                        ready_threshold;
     ngx_uint_t                        initial_ready_threshold;
@@ -208,6 +209,7 @@ typedef struct {
 
     uint32_t                          candidate_margin;
     uint32_t                          keyframe_alignment_margin;
+    uint32_t                          max_span_average;
 
     uint32_t                          ready_duration;
     uint32_t                          initial_ready_duration;
@@ -309,6 +311,13 @@ static ngx_command_t  ngx_live_segmenter_commands[] = {
       offsetof(ngx_live_segmenter_preset_conf_t, keyframe_alignment_margin),
       NULL },
 
+    { ngx_string("segmenter_max_span_average"),
+      NGX_LIVE_MAIN_CONF|NGX_LIVE_PRESET_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_msec_slot,
+      NGX_LIVE_PRESET_CONF_OFFSET,
+      offsetof(ngx_live_segmenter_preset_conf_t, max_span_average),
+      NULL },
+
     { ngx_string("segmenter_ready_threshold"),
       NGX_LIVE_MAIN_CONF|NGX_LIVE_PRESET_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_num_slot,
@@ -2391,7 +2400,13 @@ ngx_live_segmenter_get_segment_times(ngx_live_channel_t *channel,
             continue;
         }
 
-        cur_pts = (max[i] + min[i]) / 2;
+        if (max[i] - min[i] < cctx->max_span_average) {
+            cur_pts = (max[i] + min[i]) / 2;
+
+        } else {
+            cur_pts = candidates.elts[i].pts;
+        }
+
         if (target_pts == NGX_LIVE_INVALID_PTS ||
             ngx_abs_diff(cur_pts, boundary_pts) <
             ngx_abs_diff(target_pts, boundary_pts))
@@ -3629,6 +3644,8 @@ ngx_live_segmenter_channel_init(ngx_live_channel_t *channel, void *ectx)
         spcf->candidate_margin, 1000, channel->timescale);
     cctx->keyframe_alignment_margin = ngx_live_rescale_time(
         spcf->keyframe_alignment_margin, 1000, channel->timescale);
+    cctx->max_span_average = ngx_live_rescale_time(
+        spcf->max_span_average, 1000, channel->timescale);
 
     cctx->create.data = channel;
     cctx->create.handler = ngx_live_segmenter_create_handler;
@@ -3890,6 +3907,7 @@ ngx_live_segmenter_create_preset_conf(ngx_conf_t *cf)
 
     conf->candidate_margin = NGX_CONF_UNSET_MSEC;
     conf->keyframe_alignment_margin = NGX_CONF_UNSET_MSEC;
+    conf->max_span_average = NGX_CONF_UNSET_MSEC;
 
     conf->ready_threshold = NGX_CONF_UNSET_UINT;
     conf->initial_ready_threshold = NGX_CONF_UNSET_UINT;
@@ -3945,6 +3963,9 @@ ngx_live_segmenter_merge_preset_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_msec_value(conf->keyframe_alignment_margin,
                               prev->keyframe_alignment_margin, 500);
 
+    ngx_conf_merge_msec_value(conf->max_span_average,
+                              prev->max_span_average, 500);
+
     ngx_conf_merge_uint_value(conf->ready_threshold,
                               prev->ready_threshold, 150);
 

nginx-live-module/test/nginx.conf

@@ -167,7 +167,7 @@ http {
         location /ksmp_proxy/ {
             internal;
             proxy_pass http://127.0.0.1:8001/ksmp/;
-            subrequest_output_buffer_size 5m;
+            subrequest_output_buffer_size 20m;
         }
 
         # curl localhost:8001/sgts/<channel_id>/<bucket_id>/seg-<segment_index>-s<track_int_id>.ts

nginx-live-module/test/tests/audio_ends_before_video.py

@@ -0,0 +1,49 @@
+from test_base import *
+
+# EXPECTED:
+#   16 sec first video + audio
+#   2 sec first video, no audio
+#   10 sec second video + audio
+
+def test(channelId=CHANNEL_ID):
+    st = KmpSendTimestamps()
+
+    nl = setupChannelTimeline(channelId)
+
+    rv = KmpMediaFileReader(TEST_VIDEO_HIGH, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO_HIGH, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')])
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 17.5, realtime=5)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, KmpNullSender()),
+    ], st, 2, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    time.sleep(2)
+
+    initialFrameId = 1000000
+
+    rv = KmpMediaFileReader(TEST_VIDEO1, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO1, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')], initialFrameId=initialFrameId)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 10, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    # deactivate the timeline
+    nl.timeline.update(NginxLiveTimeline(id=TIMELINE_ID, end_list='on'))
+
+    testDefaultStreams(channelId, __file__)

nginx-live-module/test/tests/ref/audio_ends_before_video-hls-fmp4.txt

@@ -0,0 +1,75 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=126573,AVERAGE-BANDWIDTH=114074,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:6
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-MAP:URI="init-1-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:4971
+seg-1-svar1.m4s
+#EXTINF:7.966,
+#EXT-X-BITRATE:16215
+seg-2-svar1.m4s
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:16.342+00:00
+#EXTINF:3.200,
+#EXT-X-BITRATE:9141
+seg-3-svar1.m4s
+#EXT-X-DISCONTINUITY
+#EXT-X-MAP:URI="init-5-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:92
+seg-5-svar1.m4s
+#EXTINF:3.934,
+#EXT-X-BITRATE:123
+seg-6-svar1.m4s
+#EXTINF:1.989,
+#EXT-X-BITRATE:124
+seg-7-svar1.m4s
+#EXT-X-ENDLIST
+
+URL: /init-1-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: 44623fdc2bb6eb1183e22c022be3e765
+
+URL: /seg-1-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 5180236, MD5: 0d015341c4fbb773cfcc90ecbce81d0d
+
+URL: /seg-2-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 16014394, MD5: 309376ada6e59c0222bf0bab74028e1e
+
+URL: /seg-3-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 3732604, MD5: 12a71bf0970f3bafec0a7b800eb51b78
+
+URL: /init-5-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: ca1c220b1d37d697270c21bce220cdef
+
+URL: /seg-5-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 46063, MD5: 3cd142c193f879afc375070a3e30d431
+
+URL: /seg-6-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 61926, MD5: 8ab6413490c066bb6cbed25d13cdc626
+
+URL: /seg-7-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 32681, MD5: c03e8d0a6bd6bbb23a5844be89e1b6ea

nginx-live-module/test/tests/ref/audio_ends_before_video-hls-ts.txt

@@ -0,0 +1,65 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=144154,AVERAGE-BANDWIDTH=131383,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:3
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:5124
+seg-1-svar1.ts
+#EXTINF:7.966,
+#EXT-X-BITRATE:16612
+seg-2-svar1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:16.342+00:00
+#EXTINF:3.200,
+#EXT-X-BITRATE:9369
+seg-3-svar1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:112
+seg-5-svar1.ts
+#EXTINF:3.934,
+#EXT-X-BITRATE:145
+seg-6-svar1.ts
+#EXTINF:1.989,
+#EXT-X-BITRATE:152
+seg-7-svar1.ts
+#EXT-X-ENDLIST
+
+URL: /seg-1-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 5348788, MD5: f120a8a081f82c373c2699c75a46c728
+
+URL: /seg-2-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 16418228, MD5: 7bb081750052c1cb319753d8110c96f5
+
+URL: /seg-3-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 3826740, MD5: 61905a4e992990544087081a6855f90a
+
+URL: /seg-5-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 57716, MD5: 9aba89a2ef46c155e15b0f587bad8a61
+
+URL: /seg-6-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 72756, MD5: 57df738394e340264cde12be20eaedfa
+
+URL: /seg-7-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 39668, MD5: 4abc468def0cb3c8a592da711861fd1c

nginx-live-module/test/tests/ref/audio_slightly_before_video-hls-fmp4.txt

@@ -33,11 +33,12 @@ seg-4-svar1.m4s
 seg-5-svar1.m4s
 #EXTINF:3.800,
 seg-6-svar1.m4s
-#EXTINF:1.597,
+#EXTINF:1.195,
+#EXT-X-BITRATE:99
 seg-7-svar1.m4s
 #EXT-X-DISCONTINUITY
-#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:29.397+00:00
-#EXTINF:4.336,
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:28.995+00:00
+#EXTINF:4.738,
 #EXT-X-BITRATE:78
 seg-8-svar1.m4s
 #EXTINF:4.067,
@@ -88,25 +89,25 @@ BODY: SIZE: 21196, MD5: 5f436e7fab1fac326b00dd3a6241f25d
 
 URL: /seg-8-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 38604, MD5: aab750cf56331fe0a329bb6352e89f99
+BODY: SIZE: 38604, MD5: 783e04da1e617e155070189d87d5948a
 
 URL: /seg-9-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 52331, MD5: 63b0a3444ce7f12f62f145e7610fa14c
+BODY: SIZE: 52331, MD5: dcf10f79a419278f210037446e932256
 
 URL: /seg-10-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 44810, MD5: abc750f681293d68a11174453a26dcee
+BODY: SIZE: 44810, MD5: 2e5551724d2d29acef704d0ff062ddd2
 
 URL: /seg-11-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 45655, MD5: 073ab10b2ac2ec65a0568f4417ea5348
+BODY: SIZE: 45655, MD5: b42ef0f7ab3fc8fe4a5b4191ba0516fd
 
 URL: /seg-12-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 47467, MD5: fa6d13f86c8fda55fbeb925e8c38b616
+BODY: SIZE: 47467, MD5: 61937dbdb539af5df5a94702aca4633c
 
 URL: /seg-13-svar1.m4s
 HEADERS: 200 video/mp4
-BODY: SIZE: 51946, MD5: 3056b4cad505df810bd7ede0e5a2765b
+BODY: SIZE: 51946, MD5: 9ad4f2481e2760b0d6c19435cf5dbaf5
 

nginx-live-module/test/tests/ref/audio_slightly_before_video-hls-ts.txt

@@ -33,12 +33,12 @@ seg-5-svar1.ts
 #EXTINF:3.800,
 #EXT-X-BITRATE:119
 seg-6-svar1.ts
-#EXTINF:1.597,
-#EXT-X-BITRATE:129
+#EXTINF:1.195,
+#EXT-X-BITRATE:135
 seg-7-svar1.ts
 #EXT-X-DISCONTINUITY
-#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:29.397+00:00
-#EXTINF:4.336,
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:28.995+00:00
+#EXTINF:4.738,
 #EXT-X-BITRATE:95
 seg-8-svar1.ts
 #EXTINF:4.067,