GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-5) - WIP

- Update/fix GLMediaPlayer API doc

- GLMediaEventListener: Add event bits for all state changes to be delivered via attributesChanged(..)

- StreamWorker / Decoder Thread:
  - Use StreamWorker only !
  - Handle exceptions on StreamWorker via StreamException
  - Handles stream initialization and decoding (-> initStream(..))

- Split initGLStream(..) -> initStream(..) + initGL(GL)
  - allow initStream(..)'s implementation being executed on StreamWorker
  - allow GL initialization to be 'postponed' when stream is read,
    i.e. non blocking stream initialization (UI .. etc)

- Handle EOS via END_OF_STREAM_PTS -> pause/event

- Video: Use lock-free LFRingbuffer, similar to
  ALAudioSink (commit f18a94b3defef16e98badd6d99f2422609aa56c5)

+++

- FFMPEGDynamicLibraryBundleInfo
 - Add avcodec's:
     - avcodec_get_frame_defaults, avcodec_free_frame (54.28.0), avcodec_flush_buffers,
 - Add avutil's:
     - av_frame_unref (55.0.0)
 - Add avformat's:
     - avformat_seek_file (??)

+++

- FFMPEGMediaPlayer Native:
  - add 'snoop' video frames for a/v frame count relation.
    disabled per default, since no more needed due to ALAudioSink's
    grow-buffer usage of LFRingbuffer.

  - use sp_avcodec_free_frame if available

  - 'useRefCountedFrames=1' for libav 55.0 to cache more than one audio frame,
     not used since ALAudioSink's OpenAL usage does not require it (copies data once).
     Note: the above snooped-video frame count is used here.

  - use only one cached audio-frame (-> see above, OpenAL copies data once),
    while reusing the NIO buffer!

  - Perform OpenGL sync (glFinish) in native code!

  - find proper PTS value, i.e. either frame's PTS or DTS,
    see 'PTSStats'.

- FFMPEGMediaPlayer Java:
  - use private fields

  - simplified code due to above changes.

+++

Working Tests: MovieSimple and MovieCube

TODO-1: Fix
  - Android
  - OMXGLMediaPlayer

TODO-2:
  - Fix issue where async audio frames arrive much later than 1st video frame, i.e. around 300ms.
  - Default TextureCount .. maybe 3 ?
  - Adding Audio synchronization ?
  - Find 'truth' about correlation of audio and video PTS values,
    currently, we assume both to be unrelated ?

1 files changed, 35 insertions, 1 deletions

diff --git a/src/jogl/native/libav/ffmpeg_tool.h b/src/jogl/native/libav/ffmpeg_tool.h
index 06c3862db..76de406ae 100644
--- a/src/jogl/native/libav/ffmpeg_tool.h
+++ b/src/jogl/native/libav/ffmpeg_tool.h
@@ -52,6 +52,8 @@
 
 typedef void (APIENTRYP PFNGLTEXSUBIMAGE2DPROC) (GLenum target, GLint level, GLint xoffset, GLint yoffset, GLsizei width, GLsizei height, GLenum format, GLenum type, const GLvoid *pixels);
 typedef GLenum (APIENTRYP PFNGLGETERRORPROC) (void);
+typedef void (APIENTRYP PFNGLFLUSH) (void);
+typedef void (APIENTRYP PFNGLFINISH) (void);
 
 /**
  *  AV_TIME_BASE   1000000
@@ -68,11 +70,21 @@ typedef GLenum (APIENTRYP PFNGLGETERRORPROC) (void);
 /** Sync w/ GLMediaPlayer.STREAM_ID_AUTO */
 #define AV_STREAM_ID_AUTO -1
 
-/** Constant marking an invalid PTS, i.e. Integer.MIN_VALUE 0x80000000 {@value}. Sync w/ TextureFrame.INVALID_PTS */
+/** Default number of audio frames per video frame. Sync w/ FFMPEGMediaPlayer.AV_DEFAULT_AFRAMES. */
+#define AV_DEFAULT_AFRAMES 8
+
+/** Constant PTS marking an invalid PTS, i.e. Integer.MIN_VALUE == 0x80000000 == {@value}. Sync w/ TextureFrame.INVALID_PTS */
 #define INVALID_PTS 0x80000000
 
+/** Constant PTS marking the end of the stream, i.e. Integer.MIN_VALUE - 1 == 0x7FFFFFFF == {@value}. Sync w/ TextureFrame.END_OF_STREAM_PTS */
+#define END_OF_STREAM_PTS 0x7FFFFFFF
+
+/** Until 55.0.0 */
 #define AV_HAS_API_REQUEST_CHANNELS(pAV) (AV_VERSION_MAJOR(pAV->avcodecVersion) < 55)
 
+/** Since 55.0.0 */
+#define AV_HAS_API_REFCOUNTED_FRAMES(pAV) (AV_VERSION_MAJOR(pAV->avcodecVersion) >= 55)
+
 static inline float my_av_q2f(AVRational a){
     return a.num / (float) a.den;
 }
@@ -81,14 +93,32 @@ static inline int32_t my_av_q2i32(int64_t snum, AVRational a){
 }
 
 typedef struct {
+    void *origPtr;
+    jobject nioRef;
+    int32_t size;
+} NIOBuffer_t;
+
+typedef struct {
+    int64_t ptsError; // Number of backward PTS values (earlier than last PTS, excluding AV_NOPTS_VALUE)
+    int64_t dtsError; // Number of backward DTS values (earlier than last PTS, excluding AV_NOPTS_VALUE)
+    int64_t ptsLast;  // PTS of the last frame
+    int64_t dtsLast;  // DTS of the last frame
+} PTSStats;
+
+
+typedef struct {
     int32_t          verbose;
 
     uint32_t         avcodecVersion;
     uint32_t         avformatVersion;
     uint32_t         avutilVersion;
 
+    int32_t          useRefCountedFrames;
+
     PFNGLTEXSUBIMAGE2DPROC procAddrGLTexSubImage2D;
     PFNGLGETERRORPROC procAddrGLGetError;
+    PFNGLFLUSH procAddrGLFlush;
+    PFNGLFINISH procAddrGLFinish;
 
     AVFormatContext* pFormatCtx;
     int32_t          vid;
@@ -101,6 +131,7 @@ typedef struct {
     uint32_t         vBytesPerPixelPerPlane;
     enum PixelFormat vPixFmt;    // native decoder fmt
     int32_t          vPTS;       // msec - overall last video PTS
+    PTSStats         vPTSStats;
     int32_t          vLinesize[3];  // decoded video linesize in bytes for each plane
     int32_t          vTexWidth[3];  // decoded video tex width in bytes for each plane
 
@@ -110,6 +141,7 @@ typedef struct {
     AVCodecContext*  pACodecCtx;
     AVCodec*         pACodec;
     AVFrame**        pAFrames;
+    NIOBuffer_t*     pANIOBuffers;
     int32_t          aFrameCount;
     int32_t          aFrameCurrent;
     int32_t          aSampleRate;
@@ -117,6 +149,8 @@ typedef struct {
     int32_t          aFrameSize;
     enum AVSampleFormat aSampleFmt; // native decoder fmt
     int32_t          aPTS;       // msec - overall last audio PTS
+    PTSStats         aPTSStats;
+    int32_t          aFramesPerVideoFrame; // is 'snooped'
 
     float            fps;        // frames per seconds
     int32_t          bps_stream; // bits per seconds