From d0e01cb5c0ec3e48b8a9b9b79a7795b214c6e3ea Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Sat, 24 Aug 2013 17:56:49 +0200 Subject: GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-6) - DONE Multithreaded decoding and API should be considered stable by now, minor changes may apply if Android/OMX impl. requires it. We still need to solve TODO's as listed below, copied from 474ce65081ecd452215bc07ab866666cb11ca8b1. +++ - *TextureFrame OO changes: - TextureFrame extends TimeFrameI - GLMediaPlayerImpl* - Adapt to Ringbuffer changes of GlueGen commit f9f881e59c78e3036cb3f956bc97cfc3197f620d - Fix impl. method's API doc - getNextTextureImpl(..) returns video PTS - Fix audio-only playback - frame dropping shall only happen if: - previous frame has not been dropped - frame is too later - one decoded frame is already available - Don't block for decoder anymore: - nextFrame = "videoFramesDecoded.getBlocking() -> videoFramesDecoded.get()"; No 'next decoded frame avail' only could mean: - slow decoding/hardware - slow transport hence we shall not block rendering. - Add DEBUG output if using last frame - Add integer property 'jogl.debug.GLMediaPlayer.StreamWorker.delay' in milliseconds to simulate slow decoding, i.e. delay is added in StreamWorker after decoding before pushing new frame to Ringbuffer. - FFMPEGMediaPlayer: - audioFrameLimitWithVideo 128 -> 64 - audioFrameLimitAudioOnly 128 -> 32 - uses AudioSink's 'enqueueData(int pts, ByteBuffer bytes, int byteCount)' - fixes for audio-only playback +++ Working Tests: MovieSimple and MovieCube TODO-1: Fix - Android - OMXGLMediaPlayer TODO-2: - Fix issue where async audio frames arrive much later than 1st video frame, i.e. around 300ms. - Default TextureCount .. maybe 3 ? - Adding Audio synchronization ? - Find 'truth' about correlation of audio and video PTS values, currently, we assume both to be unrelated ? --- .../opengl/util/av/impl/FFMPEGMediaPlayer.java | 43 ++++++++++------------ .../opengl/util/av/impl/OMXGLMediaPlayer.java | 5 ++- 2 files changed, 23 insertions(+), 25 deletions(-) (limited to 'src/jogl/classes/jogamp/opengl/util/av/impl') diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index f1213d751..a800f2a31 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -40,6 +40,7 @@ import javax.media.opengl.GLException; import com.jogamp.common.util.VersionNumber; import com.jogamp.gluegen.runtime.ProcAddressTable; +import com.jogamp.opengl.util.TimeFrameI; import com.jogamp.opengl.util.GLPixelStorageModes; import com.jogamp.opengl.util.av.AudioSink; import com.jogamp.opengl.util.av.AudioSink.AudioDataFormat; @@ -169,7 +170,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { /** Initial audio frame count, ALAudioSink may grow buffer! */ private int initialAudioFrameCount = AV_DEFAULT_AFRAMES; private final int audioFrameGrowAmount = 8; - private final int audioFrameLimit = 128; + private final int audioFrameLimitWithVideo = 64; // 128; + private final int audioFrameLimitAudioOnly = 32; // 64; private SampleFormat aSampleFmt = null; private AudioSink.AudioDataFormat avChosenAudioFormat; private AudioSink.AudioDataFormat sinkChosenAudioFormat; @@ -235,6 +237,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if(null == audioSink) { throw new GLException("AudioSink null"); } + final int audioFrameLimit; if( null != gl ) { final GLContextImpl ctx = (GLContextImpl)gl.getContext(); AccessController.doPrivileged(new PrivilegedAction() { @@ -247,7 +250,10 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { setGLFuncs0(moviePtr, procAddrGLTexSubImage2D, procAddrGLGetError, procAddrGLFlush, procAddrGLFinish); return null; } } ); - } + audioFrameLimit = audioFrameLimitWithVideo; + } else { + audioFrameLimit = audioFrameLimitAudioOnly; + } sinkChosenAudioFormat = audioSink.initSink(avChosenAudioFormat, initialAudioFrameCount, audioFrameGrowAmount, audioFrameLimit); if(DEBUG) { @@ -303,7 +309,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { */ private void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane, int lSz0, int lSz1, int lSz2, - int tWd0, int tWd1, int tWd2, + int tWd0, int tWd1, int tWd2, int tH, int audioFrameCount, int sampleFmt, int sampleRate, int channels) { vPixelFmt = PixelFormat.valueOf(pixFmt); vPlanes = planes; @@ -320,7 +326,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { // w*h + 2 ( w/2 * h/2 ) // w*h + w*h/2 // 2*w/2 * h - texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = height; + texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = tH; break; // case PIX_FMT_YUYV422: case RGB24: @@ -329,7 +335,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { case RGBA: case ABGR: case BGRA: - texWidth = vTexWidth[0]; texHeight = height; + texWidth = vTexWidth[0]; texHeight = tH; break; default: // FIXME: Add more formats ! throw new RuntimeException("Unsupported pixelformat: "+vPixelFmt); @@ -487,42 +493,33 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { + protected final int getNextTextureImpl(GL gl, TextureFrame nextFrame) { if(0==moviePtr) { throw new GLException("FFMPEG native instance null"); } - int vPTS = TextureFrame.INVALID_PTS; - if( null != nextFrame ) { + int vPTS = TimeFrameI.INVALID_PTS; + if( null != gl ) { final Texture tex = nextFrame.getTexture(); tex.enable(gl); tex.bind(gl); } /** Try decode up to 10 packets to find one containing video. */ - for(int i=0; TextureFrame.INVALID_PTS == vPTS && 10 > i; i++) { + for(int i=0; TimeFrameI.INVALID_PTS == vPTS && 10 > i; i++) { vPTS = readNextPacket0(moviePtr, textureTarget, textureFormat, textureType); } - if( TextureFrame.INVALID_PTS != vPTS ) { - if( null != nextFrame ) { - nextFrame.setPTS(vPTS); - } - return true; - } else { - return false; + if( null != nextFrame ) { + nextFrame.setPTS(vPTS); } - } + return vPTS; + } private final void pushSound(ByteBuffer sampleData, int data_size, int audio_pts) { setFirstAudioPTS2SCR( audio_pts ); if( 1.0f == playSpeed || audioSinkPlaySpeedSet ) { - audioSink.enqueueData( new AudioSink.AudioFrame(sampleData, data_size, audio_pts ) ); + audioSink.enqueueData( audio_pts, sampleData, data_size); } } - - private final int getBytesPerMS(int time) { - final int bytesPerSample = sinkChosenAudioFormat.sampleSize >>> 3; // /8 - return time * ( sinkChosenAudioFormat.channelCount * bytesPerSample * ( sinkChosenAudioFormat.sampleRate / 1000 ) ); - } private static native int getAvUtilVersion0(); private static native int getAvFormatVersion0(); diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java index e1b773e9b..faa6a56c4 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java @@ -164,12 +164,13 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { + protected int getNextTextureImpl(GL gl, TextureFrame nextFrame) { if(0==moviePtr) { throw new GLException("OMX native instance null"); } final int nextTex = _getNextTextureID(moviePtr, true); if(0 < nextTex) { + // FIXME set pts ! /* FIXME final TextureSequence.TextureFrame eglImgTex = texFrameMap.get(new Integer(_getNextTextureID(moviePtr, blocking))); @@ -177,7 +178,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { lastTex = eglImgTex; } */ } - return true; + return 0; // FIXME: return pts } private String replaceAll(String orig, String search, String repl) { -- cgit v1.2.3