aboutsummaryrefslogtreecommitdiffstats
path: root/src/jogl/native/libav/ffmpeg_impl_template.c
diff options
context:
space:
mode:
authorSven Gothel <[email protected]>2023-10-15 07:06:53 +0200
committerSven Gothel <[email protected]>2023-10-15 07:06:53 +0200
commitb1956113f5601b0cc6ac525d3918a0dfa8d240af (patch)
treeb7365f799d5bdad00f8b0f60632884175cff4ef1 /src/jogl/native/libav/ffmpeg_impl_template.c
parent86210fde931400ff6f1b0a2da48ca031a957df8d (diff)
Bug 1472: Enhance GLMediaPlayer AV Sync: Utilize SCR aware audio PTS used as master-clock, enabling proper AV sync w/ untouched audio
We can finally utilize the added pass through audio PTS, see commits - GlueGen 52725b4c6525487f93407f529dc0a758b387a4fc - JOAL 12029f1ec1d8afa576e1ac61655f318cc37c1d16 This enables us to use the audio PTS as the master-clock and adjust video to the untouched audio. In case no audio is selected/playing or audio is muted, we sync merely on the system-clock (SCR) w/o audio. AV granularity is 22ms, however, since the ALAudioSink PTS may be a little late, it renders even a slightly better sync in case of too early audio (d_apts < 0). Since video frames are sync'ed to audio, the resync procedure may result in a hysteresis swinging into sync. This might be notable at start and when resumed audio or after seek. We leave the audio frames untouched to reduce processing burden and allow non-disrupted listening. Passed AV sync tests - Five-minute-sync-test.mp4 - Audio-Video-Sync-Test-Calibration-23.98fps-24fps.mp4 - Audio-Video-Sync-Test-2.mkv
Diffstat (limited to 'src/jogl/native/libav/ffmpeg_impl_template.c')
-rw-r--r--src/jogl/native/libav/ffmpeg_impl_template.c66
1 files changed, 44 insertions, 22 deletions
diff --git a/src/jogl/native/libav/ffmpeg_impl_template.c b/src/jogl/native/libav/ffmpeg_impl_template.c
index 385635fa3..e37347e3a 100644
--- a/src/jogl/native/libav/ffmpeg_impl_template.c
+++ b/src/jogl/native/libav/ffmpeg_impl_template.c
@@ -1145,20 +1145,16 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0)
return;
}
{
- const int32_t bytesPerPixel = ( pAV->vBitsPerPixel + 7 ) / 8 ;
- if(1 == pAV->vBufferPlanes) {
- pAV->vBytesPerPixelPerPlane = bytesPerPixel;
- } else {
- pAV->vBytesPerPixelPerPlane = 1;
- }
int32_t vLinesize[4];
if( pAV->vBufferPlanes > 1 ) {
+ pAV->vBytesPerPixelPerPlane = 1;
for(i=0; i<pAV->vBufferPlanes; i++) {
// FIXME: Libav Binary compatibility! JAU01
vLinesize[i] = pAV->pVFrame->linesize[i];
pAV->vTexWidth[i] = vLinesize[i] / pAV->vBytesPerPixelPerPlane ;
}
} else {
+ pAV->vBytesPerPixelPerPlane = ( pAV->vBitsPerPixel + 7 ) / 8 ;
vLinesize[0] = pAV->pVFrame->linesize[0];
if( pAV->vPixFmt == AV_PIX_FMT_YUYV422 ||
pAV->vPixFmt == AV_PIX_FMT_UYVY422 )
@@ -1298,21 +1294,42 @@ JNIEXPORT jint JNICALL FF_FUNC(readNextPacket0)
fprintf(stderr, "data %d \n", pAV->aFrameSize);
#endif
- const AVRational time_base = pAV->pAStream->time_base;
+ // FIXME: Libav Binary compatibility! JAU01
+ const AVRational pts_time_base = pAV->pAStream->time_base;
const int64_t pkt_pts = pAFrameCurrent->pts;
- if( 0 == frameCount && AV_NOPTS_VALUE != pkt_pts ) { // 1st frame only, discard invalid PTS ..
- pAV->aPTS = my_av_q2i32( pkt_pts * 1000, time_base);
+ const int64_t pkt_dts = pAFrameCurrent->pkt_dts;
+ const int64_t fix_pts = evalPTS(&pAV->aPTSStats, pkt_pts, pkt_dts);
+ const int64_t best_pts = pAFrameCurrent->best_effort_timestamp;
+ if( AV_NOPTS_VALUE != best_pts ) {
+ pAV->aPTS = my_av_q2i32( best_pts * 1000, pts_time_base);
+ } else if( AV_NOPTS_VALUE != fix_pts ) {
+ pAV->aPTS = my_av_q2i32( fix_pts * 1000, pts_time_base);
} else { // subsequent frames or invalid PTS ..
const int32_t bytesPerSample = sp_av_get_bytes_per_sample( pAV->pACodecCtx->sample_fmt );
pAV->aPTS += data_size / ( pAV->aChannels * bytesPerSample * ( pAV->aSampleRate / 1000 ) );
}
if( pAV->verbose ) {
- int32_t aDTS = my_av_q2i32( pAFrameCurrent->pkt_dts * 1000, time_base);
+ const AVRational dur_time_base = pAV->pACodecCtx->time_base;
+ const int32_t bPTS = AV_NOPTS_VALUE != best_pts ? my_av_q2i32( best_pts * 1000, pts_time_base) : 0;
+ const int32_t aPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, pts_time_base) : 0;
+ const int32_t aDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, pts_time_base) : 0;
+
+ const double frame_delay_d = av_q2d(dur_time_base) * 1000.0;
+ const double frame_repeat_d = pAFrameCurrent->repeat_pict * (frame_delay_d * 0.5);
+
+ const int32_t frame_delay_i = my_av_q2i32(1000, dur_time_base);
+ const int32_t frame_repeat_i = pAFrameCurrent->repeat_pict * (frame_delay_i / 2);
- fprintf(stderr, "A pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], f# %d, aFrame %d/%d %p, dataPtr %p, dataSize %d\n",
- pAV->aPTS, pkt_pts, aDTS, pAFrameCurrent->pkt_dts, frameCount,
- pAV->aFrameCurrent, pAV->aFrameCount, pAFrameCurrent, pAFrameCurrent->data[0], data_size);
+ const char * warn = frame_repeat_i > 0 ? "REPEAT" : "NORMAL" ;
+
+ fprintf(stderr, "A fix_pts %d, best %d [%"PRId64"], pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf ms + r %lf = %lf ms), i(%d ms + r %d = %d ms) - %s - f# %d, aFrame %d/%d %p, dataPtr %p, dataSize %d\n",
+ pAV->aPTS, bPTS, best_pts, aPTS, pkt_pts, aDTS, pkt_dts,
+ frame_delay_d, frame_repeat_d, (frame_delay_d + frame_repeat_d),
+ frame_delay_i, frame_repeat_i, (frame_delay_i + frame_repeat_i), warn, frameCount,
+ pAV->aFrameCurrent, pAV->aFrameCount, pAFrameCurrent, pAFrameCurrent->data[0], data_size);
+ // fflush(NULL);
}
+
if( NULL != env ) {
void* data_ptr = pAFrameCurrent->data[0]; // default
@@ -1426,27 +1443,32 @@ JNIEXPORT jint JNICALL FF_FUNC(readNextPacket0)
}
// FIXME: Libav Binary compatibility! JAU01
- const AVRational time_base = pAV->pVStream->time_base;
+ const AVRational pts_time_base = pAV->pVStream->time_base;
const int64_t pkt_pts = pAV->pVFrame->pts;
const int64_t pkt_dts = pAV->pVFrame->pkt_dts;
const int64_t fix_pts = evalPTS(&pAV->vPTSStats, pkt_pts, pkt_dts);
- if( AV_NOPTS_VALUE != fix_pts ) { // discard invalid PTS ..
- pAV->vPTS = my_av_q2i32( fix_pts * 1000, time_base);
+ const int64_t best_pts = pAV->pVFrame->best_effort_timestamp;
+ if( AV_NOPTS_VALUE != best_pts ) {
+ pAV->vPTS = my_av_q2i32( best_pts * 1000, pts_time_base);
+ } else if( AV_NOPTS_VALUE != fix_pts ) { // discard invalid PTS ..
+ pAV->vPTS = my_av_q2i32( fix_pts * 1000, pts_time_base);
}
if( pAV->verbose ) {
- const int32_t vPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, time_base) : 0;
- const int32_t vDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, time_base) : 0;
+ const AVRational dur_time_base = pAV->pVCodecCtx->time_base;
+ const int32_t bPTS = AV_NOPTS_VALUE != best_pts ? my_av_q2i32( best_pts * 1000, pts_time_base) : 0;
+ const int32_t vPTS = AV_NOPTS_VALUE != pkt_pts ? my_av_q2i32( pkt_pts * 1000, pts_time_base) : 0;
+ const int32_t vDTS = AV_NOPTS_VALUE != pkt_dts ? my_av_q2i32( pkt_dts * 1000, pts_time_base) : 0;
- const double frame_delay_d = av_q2d(pAV->pVCodecCtx->time_base);
+ const double frame_delay_d = av_q2d(dur_time_base);
const double frame_repeat_d = pAV->pVFrame->repeat_pict * (frame_delay_d * 0.5);
- const int32_t frame_delay_i = my_av_q2i32(1000, pAV->pVCodecCtx->time_base);
+ const int32_t frame_delay_i = my_av_q2i32(1000, dur_time_base);
const int32_t frame_repeat_i = pAV->pVFrame->repeat_pict * (frame_delay_i / 2);
const char * warn = frame_repeat_i > 0 ? "REPEAT" : "NORMAL" ;
- fprintf(stderr, "V fix_pts %d, pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf s + r %lf = %lf s), i(%d ms + r %d = %d ms) - %s - f# %d, data %p, lsz %d\n",
- pAV->vPTS, vPTS, pkt_pts, vDTS, pkt_dts,
+ fprintf(stderr, "V fix_pts %d, best %d [%"PRId64"], pts %d [pkt_pts %"PRId64"], dts %d [pkt_dts %"PRId64"], time d(%lf s + r %lf = %lf s), i(%d ms + r %d = %d ms) - %s - f# %d, data %p, lsz %d\n",
+ pAV->vPTS, bPTS, best_pts, vPTS, pkt_pts, vDTS, pkt_dts,
frame_delay_d, frame_repeat_d, (frame_delay_d + frame_repeat_d),
frame_delay_i, frame_repeat_i, (frame_delay_i + frame_repeat_i), warn, frameCount,
pAV->pVFrame->data[0], pAV->pVFrame->linesize[0]);