GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-6) - DONE

Multithreaded decoding and API should be considered stable by now,
minor changes may apply if Android/OMX impl. requires it.

We still need to solve TODO's as listed below, copied from 474ce65081ecd452215bc07ab866666cb11ca8b1.

+++

- *TextureFrame OO changes:
  - TextureFrame extends TimeFrameI

- GLMediaPlayerImpl*
  - Adapt to Ringbuffer changes of GlueGen commit f9f881e59c78e3036cb3f956bc97cfc3197f620d

  - Fix impl. method's API doc
    - getNextTextureImpl(..) returns video PTS

  - Fix audio-only playback

  - frame dropping shall only happen if:
     - previous frame has not been dropped
     - frame is too later
     - one decoded frame is already available

  - Don't block for decoder anymore:
     - nextFrame = "videoFramesDecoded.getBlocking() -> videoFramesDecoded.get()";
     No 'next decoded frame avail' only could mean:
       - slow decoding/hardware
       - slow transport
     hence we shall not block rendering.

  - Add DEBUG output if using last frame

  - Add integer property 'jogl.debug.GLMediaPlayer.StreamWorker.delay' in milliseconds
    to simulate slow decoding, i.e. delay is added in StreamWorker after decoding
    before pushing new frame to Ringbuffer.

  - FFMPEGMediaPlayer:
    - audioFrameLimitWithVideo 128 -> 64
    - audioFrameLimitAudioOnly 128 -> 32
    - uses AudioSink's 'enqueueData(int pts, ByteBuffer bytes, int byteCount)'
    - fixes for audio-only playback

+++

Working Tests: MovieSimple and MovieCube

TODO-1: Fix
  - Android
  - OMXGLMediaPlayer

TODO-2:
  - Fix issue where async audio frames arrive much later than 1st video frame, i.e. around 300ms.
  - Default TextureCount .. maybe 3 ?
  - Adding Audio synchronization ?
  - Find 'truth' about correlation of audio and video PTS values,
    currently, we assume both to be unrelated ?

6 files changed, 68 insertions, 77 deletions

diff --git a/src/test/com/jogamp/opengl/test/android/MovieCubeActivity0.java b/src/test/com/jogamp/opengl/test/android/MovieCubeActivity0.java
index c9acab64b..783742fec 100644
--- a/src/test/com/jogamp/opengl/test/android/MovieCubeActivity0.java
+++ b/src/test/com/jogamp/opengl/test/android/MovieCubeActivity0.java
@@ -107,8 +107,8 @@ public class MovieCubeActivity0 extends NewtBaseActivity {
     
                 @Override
                 public void attributesChanged(final GLMediaPlayer mp, int event_mask, long when) {
-                    System.err.println("Player AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
-                    System.err.println("Player State: "+mp);
+                    System.err.println("MovieCubeActivity0 AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
+                    System.err.println("MovieCubeActivity0 State: "+mp);
                     if( 0 != ( GLMediaEventListener.EVENT_CHANGE_INIT & event_mask ) ) {
                         glWindowMain.addGLEventListener(demoMain);
                         anim.setUpdateFPSFrames(60, System.err);
diff --git a/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity0.java b/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity0.java
index d4cb226f2..467ad1e75 100644
--- a/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity0.java
+++ b/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity0.java
@@ -104,8 +104,8 @@ public class MovieSimpleActivity0 extends NewtBaseActivity {
             
            @Override
            public void attributesChanged(GLMediaPlayer mp, int event_mask, long when) {
-               System.err.println("Player AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
-               System.err.println("Player State: "+mp);
+               System.err.println("MovieSimpleActivity0 AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
+               System.err.println("MovieSimpleActivity0 State: "+mp);
                if( 0 != ( GLMediaEventListener.EVENT_CHANGE_INIT & event_mask ) ) {
                    glWindowMain.addGLEventListener(demoMain);
                    anim.setUpdateFPSFrames(60, System.err);
diff --git a/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity1.java b/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity1.java
index 4e86883e4..84e691e76 100644
--- a/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity1.java
+++ b/src/test/com/jogamp/opengl/test/android/MovieSimpleActivity1.java
@@ -141,8 +141,8 @@ public class MovieSimpleActivity1 extends NewtBaseActivity {
             
            @Override
            public void attributesChanged(GLMediaPlayer mp, int event_mask, long when) {
-               System.err.println("Player AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
-               System.err.println("Player State: "+mp);
+               System.err.println("MovieSimpleActivity1 AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
+               System.err.println("MovieSimpleActivity1 State: "+mp);
                if( 0 != ( GLMediaEventListener.EVENT_CHANGE_INIT & event_mask ) ) {
                    glWindowMain.addGLEventListener(demoMain);
                    anim.setUpdateFPSFrames(60, System.err);
diff --git a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/TextureSequenceCubeES2.java b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/TextureSequenceCubeES2.java
index 4172a2c20..e38b9c6e3 100644
--- a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/TextureSequenceCubeES2.java
+++ b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/TextureSequenceCubeES2.java
@@ -52,6 +52,7 @@ import com.jogamp.opengl.util.glsl.ShaderState;
 import com.jogamp.opengl.util.texture.Texture;
 import com.jogamp.opengl.util.texture.TextureCoords;
 import com.jogamp.opengl.util.texture.TextureSequence;
+import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame;
 
 public class TextureSequenceCubeES2 implements GLEventListener {
     public TextureSequenceCubeES2 (TextureSequence texSource, boolean innerCube, float zoom0, float rotx, float roty) {
@@ -188,7 +189,11 @@ public class TextureSequenceCubeES2 implements GLEventListener {
     public void init(GLAutoDrawable drawable) {
         GL2ES2 gl = drawable.getGL().getGL2ES2();
         System.err.println(JoglVersion.getGLInfo(gl, null));
-        final Texture tex= texSeq.getLastTexture().getTexture();
+        final TextureFrame frame = texSeq.getLastTexture();
+        if( null == frame ) {
+            return;
+        }
+        final Texture tex= frame.getTexture();
         
         final boolean useExternalTexture = GLES2.GL_TEXTURE_EXTERNAL_OES == tex.getTarget();
         if(useExternalTexture && !gl.isExtensionAvailable("GL_OES_EGL_image_external")) {
@@ -315,20 +320,22 @@ public class TextureSequenceCubeES2 implements GLEventListener {
     
     
     private void reshapePMV(int width, int height) {
-        pmvMatrix.glMatrixMode(GLMatrixFunc.GL_PROJECTION);
-        pmvMatrix.glLoadIdentity();
-        if(!innerCube) {
-            pmvMatrix.gluPerspective(45.0f, (float)width / (float)height, 1f, 10.0f);
-            nearPlaneNormalized = 1f/(100f-1f);
-        } else {
-            pmvMatrix.glOrthof(-1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 10.0f);
-            nearPlaneNormalized = 0f;
+        if(null != pmvMatrix) {
+            pmvMatrix.glMatrixMode(GLMatrixFunc.GL_PROJECTION);
+            pmvMatrix.glLoadIdentity();
+            if(!innerCube) {
+                pmvMatrix.gluPerspective(45.0f, (float)width / (float)height, 1f, 10.0f);
+                nearPlaneNormalized = 1f/(100f-1f);
+            } else {
+                pmvMatrix.glOrthof(-1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 10.0f);
+                nearPlaneNormalized = 0f;
+            }
+            System.err.println("XXX0: Perspective nearPlaneNormalized: "+nearPlaneNormalized);
+    
+            pmvMatrix.glMatrixMode(GLMatrixFunc.GL_MODELVIEW);
+            pmvMatrix.glLoadIdentity();
+            pmvMatrix.glTranslatef(0, 0, zoom);
         }
-        System.err.println("XXX0: Perspective nearPlaneNormalized: "+nearPlaneNormalized);
-
-        pmvMatrix.glMatrixMode(GLMatrixFunc.GL_MODELVIEW);
-        pmvMatrix.glLoadIdentity();
-        pmvMatrix.glTranslatef(0, 0, zoom);
     }
 
 
diff --git a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieCube.java b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieCube.java
index c48c53189..7fa55f861 100644
--- a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieCube.java
+++ b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieCube.java
@@ -60,7 +60,7 @@ import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame;
 /**
  * Simple cube movie player w/ aspect ration true projection on a cube. 
  */
-public class MovieCube implements GLEventListener, GLMediaEventListener {
+public class MovieCube implements GLEventListener {
     private static boolean waitForKey = false;
     private final float zoom0, rotx, roty;
     private TextureSequenceCubeES2 cube=null;
@@ -89,11 +89,9 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
         this.rotx = rotx;
         this.roty = roty;
         mPlayer = GLMediaPlayerFactory.createDefault();
-        mPlayer.addEventListener(this);        
     }
 
     public void initStream(URI streamLoc, int vid, int aid, int textureCount) {
-        mPlayer.addEventListener(this);
         mPlayer.initStream(streamLoc, vid, aid, textureCount);
         System.out.println("pC.1b "+mPlayer);
     }
@@ -161,16 +159,6 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
     };
     
     @Override
-    public void attributesChanged(GLMediaPlayer mp, int event_mask, long when) {
-        System.out.println("attributesChanges: "+mp+", 0x"+Integer.toHexString(event_mask)+", when "+when);        
-    }
-
-    @Override
-    public void newFrameAvailable(GLMediaPlayer mp, TextureFrame newFrame, long when) {
-        // System.out.println("newFrameAvailable: "+mp+", when "+when);
-    }
-
-    @Override
     public void init(GLAutoDrawable drawable) {
         if(null == mPlayer) {
             throw new InternalError("mPlayer null");
@@ -179,8 +167,7 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
             throw new IllegalStateException("mPlayer not in state initialized: "+mPlayer);
         }
         if( GLMediaPlayer.STREAM_ID_NONE == mPlayer.getVID() ) {
-            System.err.println("MovieCube: No VID/stream selected - no GL: "+mPlayer);
-            return;
+            // throw new IllegalStateException("mPlayer has no VID/stream selected: "+mPlayer);
         }
         GL2ES2 gl = drawable.getGL().getGL2ES2();
         System.err.println(JoglVersion.getGLInfo(gl, null));
@@ -203,7 +190,7 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
         }
         cube.init(drawable);
         mPlayer.play();
-        System.out.println("pStart "+mPlayer);
+        System.out.println("play.0 "+mPlayer);
 
         boolean added;
         final Object upstreamWidget = drawable.getUpstreamWidget();
@@ -245,7 +232,6 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
             System.err.println( mPlayer.getPerfString() );
             lastPerfPos = currentPos;  
         }
-        
         cube.display(drawable);
     }
 
@@ -353,8 +339,8 @@ public class MovieCube implements GLEventListener, GLMediaEventListener {
 
             @Override
             public void attributesChanged(final GLMediaPlayer mp, int event_mask, long when) {
-                System.err.println("Player AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
-                System.err.println("Player State: "+mp);
+                System.err.println("MovieCube AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
+                System.err.println("MovieCube State: "+mp);
                 if( 0 != ( GLMediaEventListener.EVENT_CHANGE_SIZE & event_mask ) && origSize ) {
                     window.setSize(mp.getWidth(), mp.getHeight());
                 }
diff --git a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieSimple.java b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieSimple.java
index ecf95f069..672500b1b 100644
--- a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieSimple.java
+++ b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/av/MovieSimple.java
@@ -75,7 +75,7 @@ import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame;
 /**
  * Simple planar movie player w/ orthogonal 1:1 projection. 
  */
-public class MovieSimple implements GLEventListener, GLMediaEventListener {
+public class MovieSimple implements GLEventListener {
     public static final int EFFECT_NORMAL                  =    0;
     public static final int EFFECT_GRADIENT_BOTTOM2TOP     = 1<<1;
     public static final int EFFECT_TRANSPARENT             = 1<<3; 
@@ -234,12 +234,10 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
         mPlayerShared = true;
         mPlayerExternal = true;
         mPlayer = sharedMediaPlayer;
-        mPlayer.addEventListener(this);
         System.out.println("pC.2 shared "+mPlayerShared+", "+mPlayer);
     }
     
     public void initStream(URI streamLoc, int vid, int aid, int textureCount) {
-        mPlayer.addEventListener(this);
         mPlayer.initStream(streamLoc, vid, aid, textureCount);
         System.out.println("pC.1b "+mPlayer);
     }
@@ -263,16 +261,6 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
         this.alpha = alpha;
     }    
 
-    @Override
-    public void attributesChanged(GLMediaPlayer mp, int event_mask, long when) {
-        System.out.println("attributesChanges: "+mp+", 0x"+Integer.toHexString(event_mask)+", when "+when);        
-    }
-
-    @Override
-    public void newFrameAvailable(GLMediaPlayer mp, TextureFrame newFrame, long when) {
-        // System.out.println("newFrameAvailable: "+mp+", when "+when);        
-    }
-
     private void initShader(GL2ES2 gl) {
         // Create & Compile the shader objects
         ShaderCode rsVp = ShaderCode.create(gl, GL2ES2.GL_VERTEX_SHADER, MovieSimple.class, 
@@ -316,8 +304,7 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
             throw new IllegalStateException("mPlayer not in state initialized: "+mPlayer);
         }
         if( GLMediaPlayer.STREAM_ID_NONE == mPlayer.getVID() ) {
-            System.err.println("MovieSimple: No VID/stream selected - no GL: "+mPlayer);
-            return;
+            throw new IllegalStateException("mPlayer has no VID/stream selected: "+mPlayer);
         }
         zoom0 =  orthoProjection ? 0f : -2.5f;
         zoom1 = orthoProjection ? 0f : -5f;
@@ -338,8 +325,12 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
             if(!mPlayerShared) {
                 mPlayer.initGL(gl);
             }
-            tex = mPlayer.getLastTexture().getTexture();
             System.out.println("p1 "+mPlayer+", shared "+mPlayerShared);
+            final TextureFrame frame = mPlayer.getLastTexture();
+            if( null == frame ) {
+                throw new InternalError("XXX: "+mPlayer);
+            }
+            tex = mPlayer.getLastTexture().getTexture();
             useExternalTexture = GLES2.GL_TEXTURE_EXTERNAL_OES == tex.getTarget();
             if(useExternalTexture && !gl.isExtensionAvailable("GL_OES_EGL_image_external")) {
                 throw new GLException("GL_OES_EGL_image_external requested but not available");
@@ -415,10 +406,9 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
             interleavedVBO.addGLSLSubArray("mgl_MultiTexCoord", 2, GL.GL_ARRAY_BUFFER);
             
             final FloatBuffer ib = (FloatBuffer)interleavedVBO.getBuffer();
-            final TextureCoords tc = tex.getImageTexCoords();                                   
-            final float aspect = tex.getAspectRatio();
+            final TextureCoords tc = tex.getImageTexCoords();
             System.err.println("XXX0: "+tc);
-            System.err.println("XXX0: tex aspect: "+aspect);
+            System.err.println("XXX0: tex aspect: "+tex.getAspectRatio());
             System.err.println("XXX0: tex y-flip: "+tex.getMustFlipVertically());
             
              // left-bottom
@@ -472,7 +462,7 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
 
         if(!mPlayerShared) {
             mPlayer.play();
-            System.out.println("pStart "+mPlayer);
+            System.out.println("play.0 "+mPlayer);
         }        
         startTime = System.currentTimeMillis();
         
@@ -493,9 +483,6 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
             gl.setSwapInterval(swapInterval); // in case switching the drawable (impl. may bound attribute there)
         }
         if(null == mPlayer) { return; }
-        if( GLMediaPlayer.STREAM_ID_NONE == mPlayer.getVID() ) {
-            return;
-        }
         winWidth = width;
         winHeight = height;
                 
@@ -535,7 +522,6 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
         System.out.println("pD.1 "+mPlayer);        
         GL2ES2 gl = drawable.getGL().getGL2ES2();
         if( null != mPlayer ) {
-            mPlayer.removeEventListener(this);
             if(!mPlayerExternal) {
                 mPlayer.destroy(gl);
             }
@@ -563,11 +549,7 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
         if( currentPos - lastPerfPos > 2000 ) {
             System.err.println( mPlayer.getPerfString() );
             lastPerfPos = currentPos;  
-        }
-        
-        if( GLMediaPlayer.STREAM_ID_NONE == mPlayer.getVID() ) {
-            return;
-        }
+        }        
                 
         GL2ES2 gl = drawable.getGL().getGL2ES2();        
 
@@ -721,31 +703,47 @@ public class MovieSimple implements GLEventListener, GLMediaEventListener {
             anim.start();
 
             ms.mPlayer.addEventListener(new GLMediaEventListener() {
+                void destroyWindow() {
+                    new Thread() {
+                        public void run() {
+                            window.destroy();
+                        } }.start();                    
+                }
+                
                 @Override
                 public void newFrameAvailable(GLMediaPlayer ts, TextureFrame newFrame, long when) {
                 }
     
                 @Override
                 public void attributesChanged(final GLMediaPlayer mp, int event_mask, long when) {
-                    System.err.println("Player AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
-                    System.err.println("Player State: "+mp);
+                    System.err.println("MovieSimple AttributesChanges: events_mask 0x"+Integer.toHexString(event_mask)+", when "+when);
+                    System.err.println("MovieSimple State: "+mp);
                     if( 0 != ( GLMediaEventListener.EVENT_CHANGE_SIZE & event_mask ) && origSize ) {
                         window.setSize(mp.getWidth(), mp.getHeight());
                     }
                     if( 0 != ( GLMediaEventListener.EVENT_CHANGE_INIT & event_mask ) ) {
-                        window.addGLEventListener(ms);
-                        anim.setUpdateFPSFrames(60, System.err);
-                        anim.resetFPSCounter();
+                        if( GLMediaPlayer.STREAM_ID_NONE != ms.mPlayer.getVID() ) {
+                            window.addGLEventListener(ms);
+                            anim.setUpdateFPSFrames(60, System.err);
+                            anim.resetFPSCounter();
+                        } else {
+                            try {
+                                ms.mPlayer.initGL(null);
+                            } catch (Exception e) {
+                                e.printStackTrace();
+                                destroyWindow();
+                                return;
+                            }
+                            ms.mPlayer.play();
+                            System.out.println("play.1 "+ms.mPlayer);
+                        }
                     }
                     if( 0 != ( ( GLMediaEventListener.EVENT_CHANGE_ERR | GLMediaEventListener.EVENT_CHANGE_EOS ) & event_mask ) ) {
                         final StreamException se = ms.mPlayer.getStreamException();
                         if( null != se ) {
                             se.printStackTrace();                        
                         }
-                        new Thread() {
-                            public void run() {
-                                window.destroy();
-                            } }.start();
+                        destroyWindow();
                     }
                 }
             });