FFMPEGMediaPlayer: Poor mans audio/video sync.

Signed-off-by: Xerxes Rånby <[email protected]>

2 files changed, 130 insertions, 18 deletions

diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
index cf26bbd81..2c7134a44 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
@@ -37,6 +37,7 @@ import javax.media.opengl.GL2ES2;
 import javax.media.opengl.GLException;
 
 import java.util.Arrays;
+import java.util.Queue;
 import javax.sound.sampled.*;
 
 import com.jogamp.common.util.VersionNumber;
@@ -109,11 +110,11 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
     private static final int TEMP_BUFFER_COUNT = 20;
 
     // AudioFormat parameters
-    public  static final int     SAMPLE_RATE = 22050;
+    public  static final int     SAMPLE_RATE = 44100;
     private static final int     SAMPLE_SIZE = 16;
-    private static final int     CHANNELS = 1;
+    private static final int     CHANNELS = 2;
     private static final boolean SIGNED = true;
-    private static final boolean BIG_ENDIAN = true;
+    private static final boolean BIG_ENDIAN = false;
 
     // Chunk of audio processed at one time
     public static final int BUFFER_SIZE = 1000;
@@ -130,6 +131,8 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
     private static int bufferCount;
     private static byte [] sampleData = new byte[BUFFER_SIZE];
 
+    private static int maxAvailableAudio;
+
     public static final VersionNumber avUtilVersion;
     public static final VersionNumber avFormatVersion;
     public static final VersionNumber avCodecVersion;    
@@ -158,8 +161,10 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
                     auline = (SourceDataLine) AudioSystem.getLine(info);
                     auline.open(format);
                     auline.start();
+                    maxAvailableAudio = auline.available();
                     available = true;
                 } catch (LineUnavailableException e){
+                    maxAvailableAudio = 0;
                     available = false;
                 }
             }
@@ -262,12 +267,83 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
             throw new InternalError("Unknown ProcAddressTable: "+pt.getClass().getName()+" of "+ctx.getClass().getName());
         }
     }
-    private void updateSound(byte[] sampleData, int data_size) {
+
+    private class AudioFrame {
+        final byte[] sampleData; 
+        final int data_size;
+        final int audio_pts;
+        AudioFrame(byte[] sampleData, int data_size, int audio_pts) {
+            this.sampleData=sampleData;
+            this.data_size=data_size;
+            this.audio_pts=audio_pts;
+        }
+    }
+
+    static final Queue<AudioFrame> audioFrameBuffer = new java.util.LinkedList<AudioFrame>(); 
+
+    private void updateSound(byte[] sampleData, int data_size, int audio_pts) {
+/*
+        // Visualize incomming data
+        int c=0; 
+        for(byte b: sampleData){
+            if(b<0) {
+                System.out.print(" ");
+            } else if(b<64) {
+                System.out.print("_");
+            } else if(b < 128) {
+                System.out.print("-");
+            } else if(b == 128) {
+                System.out.print("=");
+            } else if(b < 256-64) {
+                System.out.print("\"");
+            } else {
+                System.out.print("'");
+            }
+                        
+            c++;
+            if(c>=40)
+                break;                   
+        }
         System.out.println("jA");
-        if (data_size > 0) {
-             auline.write(sampleData, 0, data_size);
+*/
+
+        //TODO reduce GC
+        audioFrameBuffer.add(new AudioFrame(sampleData, data_size, audio_pts));
+        pumpAudio();
+    }
+
+    private void pumpAudio() {
+        if(auline.available()==maxAvailableAudio){
+           System.out.println("warning: audio buffer underrun");
+        }
+        while(audioFrameBuffer.peek()!=null){
+            AudioFrame a = audioFrameBuffer.peek();
+
+            // poor mans audio sync .. TODO: off thread
+            final long now = System.currentTimeMillis();
+            final long now_d = now - lastAudioTime;
+            final long pts_d = a.audio_pts - lastAudioPTS;
+            final long dt = (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ;
+           
+            System.err.println("s: pts-a "+a.audio_pts+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt); 
+            lastAudioTime = now;
+            if( (dt<audio_dt_d  ) && auline.available()>a.data_size ) {
+                audioFrameBuffer.poll(); /* remove first item from the queue */
+                int written = 0;
+                int len;
+                int data_size = a.data_size;
+                while (data_size > 0) {
+                    len = auline.write(a.sampleData, written, data_size);
+                    data_size -= len;
+                    written += len;
+                }
+                lastAudioPTS=a.audio_pts; 
+            } else {
+              break;
+            }
         }
     }
+
     private void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane,
                                    int lSz0, int lSz1, int lSz2,
                                    int tWd0, int tWd1, int tWd2) {
@@ -413,6 +489,9 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
         int pts0 = getVideoPTS0(moviePtr);
         int pts1 = seek0(moviePtr, msec);
         System.err.println("Seek: "+pts0+" -> "+msec+" : "+pts1);
+        audioFrameBuffer.clear();
+        lastAudioPTS=pts1;
+        lastVideoPTS=pts1;
         return pts1;
     }
 
@@ -421,9 +500,12 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
         return lastTex;
     }
     
+    private long lastAudioTime = 0;
+    private int lastAudioPTS = 0;
+    private static final int audio_dt_d = 400;
     private long lastVideoTime = 0;
     private int lastVideoPTS = 0;
-    private static final int dt_d = 9;
+    private static final int video_dt_d = 9;
     
     @Override
     protected TextureSequence.TextureFrame getNextTextureImpl(GL gl, boolean blocking) {
@@ -453,19 +535,23 @@ public class FFMPEGMediaPlayer extends EGLMediaPlayerImpl {
             if(blocking) {
                 // poor mans video sync .. TODO: off thread 'readNextPackage0(..)' on shared GLContext and multi textures/unit!
                 final long now = System.currentTimeMillis();
-                final long now_d = now - lastVideoTime;
-                final long pts_d = pts - lastVideoPTS;                
+                // Try sync video to audio
+                final long now_d = now - lastAudioTime;
+                final long pts_d = pts - lastAudioPTS - 444; /* hack 444 == play video 444ms ahead of audio */
+                //final long dt = Math.min(46, Math.abs( (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ) ) ;
                 final long dt = (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ;
                 lastVideoTime = now;
-                // System.err.println("s: pts-v "+pts+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt);
-                if(dt>dt_d && dt<1000 ) {
+                System.err.println("s: pts-v "+pts+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt);
+                
+                if(dt>video_dt_d && dt<1000 && auline.available()<maxAvailableAudio-10000) {
                     try {
-                        Thread.sleep(dt-dt_d);
+                        Thread.sleep(dt-video_dt_d);
                     } catch (InterruptedException e) { }
                 } /* else if(0>pts_d) {
                     System.err.println("s: pts-v "+pts+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt);
                 } */
             }
+            pumpAudio();
             lastVideoPTS = pts;
         }
         return lastTex;
diff --git a/src/jogl/native/libav/jogamp_opengl_util_av_impl_FFMPEGMediaPlayer.c b/src/jogl/native/libav/jogamp_opengl_util_av_impl_FFMPEGMediaPlayer.c
index b8ff9a6ca..bac299954 100644
--- a/src/jogl/native/libav/jogamp_opengl_util_av_impl_FFMPEGMediaPlayer.c
+++ b/src/jogl/native/libav/jogamp_opengl_util_av_impl_FFMPEGMediaPlayer.c
@@ -192,12 +192,38 @@ JNIEXPORT jboolean JNICALL Java_jogamp_opengl_util_av_impl_FFMPEGDynamicLibraryB
     return JNI_TRUE;
 }
 
-static void _updateSound(JNIEnv *env, jobject instance, char *data, int32_t data_size) {
+static void _updateSound(JNIEnv *env, jobject instance, int8_t *data, int32_t data_size, int32_t aPTS) {
     if(NULL!=env) {
-        fprintf(stderr, "nA");
         jbyteArray jbArray = (*env)->NewByteArray(env, data_size);
-        (*env)->SetByteArrayRegion(env, jbArray, 0, data_size, (jbyte*)data);
-        (*env)->CallVoidMethod(env, instance, jni_mid_updateSound, jbArray, data_size);
+        if (jbArray == NULL) {
+            fprintf(stderr, "FFMPEGMediaPlayer out of memory at native code _updateSound");
+            return; /* out of memory error thrown */
+        }
+
+/*
+        // Visualize sample waveform
+        int i;
+        for(i=0;i<40;i++){
+            int8_t b = data[i];
+            if(b<0) {
+                printf(" ");
+            } else if(b<64) {
+                printf("_");
+            } else if(b < 128) {
+                printf("-");
+            } else if(b == 128) {
+                printf("=");
+            } else if(b < 256-64) {
+                printf("\"");
+            } else {
+                printf("'");
+            }
+        }
+        printf("nA\n");
+*/
+
+        (*env)->SetByteArrayRegion(env, jbArray, 0, data_size, data);
+        (*env)->CallVoidMethod(env, instance, jni_mid_updateSound, jbArray, data_size, aPTS);
     }
 }
 
@@ -344,7 +370,7 @@ JNIEXPORT jboolean JNICALL Java_jogamp_opengl_util_av_impl_FFMPEGMediaPlayer_ini
         JoglCommon_FatalError(env, "JOGL FFMPEG: can't use %s", ClazzNameFFMPEGMediaPlayer);
     }
 
-    jni_mid_updateSound = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "updateSound", "([BI)V");
+    jni_mid_updateSound = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "updateSound", "([BII)V");
     jni_mid_updateAttributes1 = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "updateAttributes", "(IIIIIFIILjava/lang/String;Ljava/lang/String;)V");
     jni_mid_updateAttributes2 = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "updateAttributes2", "(IIIIIIIIII)V");
 
@@ -656,7 +682,7 @@ JNIEXPORT jint JNICALL Java_jogamp_opengl_util_av_impl_FFMPEGMediaPlayer_readNex
                 // TODO: Wrap audio buffer data in a com.jogamp.openal.sound3d.Buffer or similar
                 // and hand it over to the user using a suitable API.
                 // TODO: OR send the audio buffer data down to sound card directly using JOAL.
-                _updateSound(env, instance, pAV->pAFrame->data[0], data_size);
+                _updateSound(env, instance, pAV->pAFrame->data[0], data_size, pAV->aPTS);
 
                 res = 1;
             }