diff options
author | Sven Gothel <[email protected]> | 2023-10-15 07:06:53 +0200 |
---|---|---|
committer | Sven Gothel <[email protected]> | 2023-10-15 07:06:53 +0200 |
commit | b1956113f5601b0cc6ac525d3918a0dfa8d240af (patch) | |
tree | b7365f799d5bdad00f8b0f60632884175cff4ef1 /src/jogl/native/libav/ffmpeg_impl_template.c | |
parent | 86210fde931400ff6f1b0a2da48ca031a957df8d (diff) |
Bug 1472: Enhance GLMediaPlayer AV Sync: Utilize SCR aware audio PTS used as master-clock, enabling proper AV sync w/ untouched audio
We can finally utilize the added pass through audio PTS, see commits
- GlueGen 52725b4c6525487f93407f529dc0a758b387a4fc
- JOAL 12029f1ec1d8afa576e1ac61655f318cc37c1d16
This enables us to use the audio PTS as the master-clock and adjust video to the untouched audio.
In case no audio is selected/playing or audio is muted,
we sync merely on the system-clock (SCR) w/o audio.
AV granularity is 22ms, however, since the ALAudioSink PTS may be a little late,
it renders even a slightly better sync in case of too early audio (d_apts < 0).
Since video frames are sync'ed to audio, the resync procedure may result
in a hysteresis swinging into sync. This might be notable at start
and when resumed audio or after seek.
We leave the audio frames untouched to reduce processing burden
and allow non-disrupted listening.
Passed AV sync tests
- Five-minute-sync-test.mp4
- Audio-Video-Sync-Test-Calibration-23.98fps-24fps.mp4
- Audio-Video-Sync-Test-2.mkv
Diffstat (limited to 'src/jogl/native/libav/ffmpeg_impl_template.c')
-rw-r--r-- | src/jogl/native/libav/ffmpeg_impl_template.c | 66 |
1 files changed, 44 insertions, 22 deletions
diff --git a/src/jogl/native/libav/ffmpeg_impl_template.c b/src/jogl/native/libav/ffmpeg_impl_template.c index 385635fa3..e37347e3a 100644 --- a/src/jogl/native/libav/ffmpeg_impl_template.c +++ b/src/jogl/native/libav/ffmpeg_impl_template.c @@ -1145,20 +1145,16 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0) return; } { - const int32_t bytesPerPixel = ( pAV->vBitsPerPixel + 7 ) / 8 ; - if(1 == pAV->vBufferPlanes) { - pAV->vBytesPerPixelPerPlane = bytesPerPixel; - } else { - pAV->vBytesPerPixelPerPlane = 1; - } int32_t vLinesize[4]; if( pAV->vBufferPlanes > 1 ) { + pAV->vBytesPerPixelPerPlane = 1; for(i=0; i<pAV->vBufferPlanes; i++) { // FIXME: Libav Binary compatibility! JAU01 vLinesize[i] = pAV->pVFrame->linesize[i]; pAV->vTexWidth[i] = vLinesize[i] / pAV->vBytesPerPixelPerPlane ; } } else { + pAV->vBytesPerPixelPerPlane = ( pAV->vBitsPerPixel + 7 ) / 8 ; vLinesize[0] = pAV->pVFrame->linesize[0]; if( pAV->vPixFmt == AV_PIX_FMT_YUYV422 || pAV->vPixFmt == AV_PIX_FMT_UYVY422 ) @@ -1298,21 +1294,42 @@ JNIEXPORT jint JNICALL FF_FUNC(readNextPacket0) fprintf(stderr, "data %d \n", pAV->aFrameSize); #endif - const AVRational time_base = pAV->pAStream->time_base; + // FIXME: Libav Binary compatibility! JAU01 + const AVRational pts_time_base = pAV->pAStream->time_base; const int64_t pkt_pts = pAFrameCurrent->pts; - if( 0 == frameCount && AV_NOPTS_VALUE != pkt_pts ) { // 1st frame only, discard invalid PTS .. - pAV->aPTS = my_av_q2i32( pkt_pts * 1000, time_base); + const int64_t pkt_dts = pAFrameCurrent->pkt_dts; + const int64_t fix_pts = evalPTS(&pAV->aPTSStats, pkt_pts, pkt_dts); + const int64_t best_pts = pAFrameCurrent->best_effort_timestamp; + if( AV_NOPTS_VALUE != best_pts ) { + pAV->aPTS = my_av_q2i32( best_pts * 1000, pts_time_base); + } else if( AV_NOPTS_VALUE != fix_pts ) { + pAV->aPTS = my_av_q2i32( fix_pts * 1000, pts_time_base); } else { // subsequent frames or invalid PTS .. const int32_t bytesPerSample = sp_av_get_bytes_per_sample( pAV->pACodecCtx->sample_fmt ); pAV->aPTS += data_size / ( pAV->aChannels * bytesPerSample * ( pAV->aSampleRate / 1000 ) ); } if( pAV->verbose ) { - int32_t aDTS = my_av_q2i32( pAFrameCurrent->pkt_dts * 1000, time_base); + const AVRational dur_time_base = pAV->pACodecCtx->time_base; + const int32_t bPTS = AV_NOPTS_VALUE != best_pts ? my_av_q2i32( best_pts * 1000, pts_time_base) : 0; + const int32_t aPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, pts_time_base) : 0; + const int32_t aDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, pts_time_base) : 0; + + const double frame_delay_d = av_q2d(dur_time_base) * 1000.0; + const double frame_repeat_d = pAFrameCurrent->repeat_pict * (frame_delay_d * 0.5); + + const int32_t frame_delay_i = my_av_q2i32(1000, dur_time_base); + const int32_t frame_repeat_i = pAFrameCurrent->repeat_pict * (frame_delay_i / 2); - fprintf(stderr, "A pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], f# %d, aFrame %d/%d %p, dataPtr %p, dataSize %d\n", - pAV->aPTS, pkt_pts, aDTS, pAFrameCurrent->pkt_dts, frameCount, - pAV->aFrameCurrent, pAV->aFrameCount, pAFrameCurrent, pAFrameCurrent->data[0], data_size); + const char * warn = frame_repeat_i > 0 ? "REPEAT" : "NORMAL" ; + + fprintf(stderr, "A fix_pts %d, best %d [%"PRId64"], pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf ms + r %lf = %lf ms), i(%d ms + r %d = %d ms) - %s - f# %d, aFrame %d/%d %p, dataPtr %p, dataSize %d\n", + pAV->aPTS, bPTS, best_pts, aPTS, pkt_pts, aDTS, pkt_dts, + frame_delay_d, frame_repeat_d, (frame_delay_d + frame_repeat_d), + frame_delay_i, frame_repeat_i, (frame_delay_i + frame_repeat_i), warn, frameCount, + pAV->aFrameCurrent, pAV->aFrameCount, pAFrameCurrent, pAFrameCurrent->data[0], data_size); + // fflush(NULL); } + if( NULL != env ) { void* data_ptr = pAFrameCurrent->data[0]; // default @@ -1426,27 +1443,32 @@ JNIEXPORT jint JNICALL FF_FUNC(readNextPacket0) } // FIXME: Libav Binary compatibility! JAU01 - const AVRational time_base = pAV->pVStream->time_base; + const AVRational pts_time_base = pAV->pVStream->time_base; const int64_t pkt_pts = pAV->pVFrame->pts; const int64_t pkt_dts = pAV->pVFrame->pkt_dts; const int64_t fix_pts = evalPTS(&pAV->vPTSStats, pkt_pts, pkt_dts); - if( AV_NOPTS_VALUE != fix_pts ) { // discard invalid PTS .. - pAV->vPTS = my_av_q2i32( fix_pts * 1000, time_base); + const int64_t best_pts = pAV->pVFrame->best_effort_timestamp; + if( AV_NOPTS_VALUE != best_pts ) { + pAV->vPTS = my_av_q2i32( best_pts * 1000, pts_time_base); + } else if( AV_NOPTS_VALUE != fix_pts ) { // discard invalid PTS .. + pAV->vPTS = my_av_q2i32( fix_pts * 1000, pts_time_base); } if( pAV->verbose ) { - const int32_t vPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, time_base) : 0; - const int32_t vDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, time_base) : 0; + const AVRational dur_time_base = pAV->pVCodecCtx->time_base; + const int32_t bPTS = AV_NOPTS_VALUE != best_pts ? my_av_q2i32( best_pts * 1000, pts_time_base) : 0; + const int32_t vPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, pts_time_base) : 0; + const int32_t vDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, pts_time_base) : 0; - const double frame_delay_d = av_q2d(pAV->pVCodecCtx->time_base); + const double frame_delay_d = av_q2d(dur_time_base); const double frame_repeat_d = pAV->pVFrame->repeat_pict * (frame_delay_d * 0.5); - const int32_t frame_delay_i = my_av_q2i32(1000, pAV->pVCodecCtx->time_base); + const int32_t frame_delay_i = my_av_q2i32(1000, dur_time_base); const int32_t frame_repeat_i = pAV->pVFrame->repeat_pict * (frame_delay_i / 2); const char * warn = frame_repeat_i > 0 ? "REPEAT" : "NORMAL" ; - fprintf(stderr, "V fix_pts %d, pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf s + r %lf = %lf s), i(%d ms + r %d = %d ms) - %s - f# %d, data %p, lsz %d\n", - pAV->vPTS, vPTS, pkt_pts, vDTS, pkt_dts, + fprintf(stderr, "V fix_pts %d, best %d [%"PRId64"], pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf s + r %lf = %lf s), i(%d ms + r %d = %d ms) - %s - f# %d, data %p, lsz %d\n", + pAV->vPTS, bPTS, best_pts, vPTS, pkt_pts, vDTS, pkt_dts, frame_delay_d, frame_repeat_d, (frame_delay_d + frame_repeat_d), frame_delay_i, frame_repeat_i, (frame_delay_i + frame_repeat_i), warn, frameCount, pAV->pVFrame->data[0], pAV->pVFrame->linesize[0]); |