From d0e01cb5c0ec3e48b8a9b9b79a7795b214c6e3ea Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Sat, 24 Aug 2013 17:56:49 +0200 Subject: GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-6) - DONE Multithreaded decoding and API should be considered stable by now, minor changes may apply if Android/OMX impl. requires it. We still need to solve TODO's as listed below, copied from 474ce65081ecd452215bc07ab866666cb11ca8b1. +++ - *TextureFrame OO changes: - TextureFrame extends TimeFrameI - GLMediaPlayerImpl* - Adapt to Ringbuffer changes of GlueGen commit f9f881e59c78e3036cb3f956bc97cfc3197f620d - Fix impl. method's API doc - getNextTextureImpl(..) returns video PTS - Fix audio-only playback - frame dropping shall only happen if: - previous frame has not been dropped - frame is too later - one decoded frame is already available - Don't block for decoder anymore: - nextFrame = "videoFramesDecoded.getBlocking() -> videoFramesDecoded.get()"; No 'next decoded frame avail' only could mean: - slow decoding/hardware - slow transport hence we shall not block rendering. - Add DEBUG output if using last frame - Add integer property 'jogl.debug.GLMediaPlayer.StreamWorker.delay' in milliseconds to simulate slow decoding, i.e. delay is added in StreamWorker after decoding before pushing new frame to Ringbuffer. - FFMPEGMediaPlayer: - audioFrameLimitWithVideo 128 -> 64 - audioFrameLimitAudioOnly 128 -> 32 - uses AudioSink's 'enqueueData(int pts, ByteBuffer bytes, int byteCount)' - fixes for audio-only playback +++ Working Tests: MovieSimple and MovieCube TODO-1: Fix - Android - OMXGLMediaPlayer TODO-2: - Fix issue where async audio frames arrive much later than 1st video frame, i.e. around 300ms. - Default TextureCount .. maybe 3 ? - Adding Audio synchronization ? - Find 'truth' about correlation of audio and video PTS values, currently, we assume both to be unrelated ? --- .../com/jogamp/opengl/util/av/GLMediaPlayer.java | 12 +- .../opengl/util/texture/TextureSequence.java | 28 +-- .../android/av/AndroidGLMediaPlayerAPI14.java | 9 +- .../jogamp/opengl/util/av/GLMediaPlayerImpl.java | 253 +++++++++++++-------- .../jogamp/opengl/util/av/NullGLMediaPlayer.java | 41 ++-- .../opengl/util/av/impl/FFMPEGMediaPlayer.java | 43 ++-- .../opengl/util/av/impl/OMXGLMediaPlayer.java | 5 +- 7 files changed, 223 insertions(+), 168 deletions(-) (limited to 'src/jogl/classes') diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java index 726eddb01..02fbd721c 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java @@ -35,6 +35,7 @@ import javax.media.opengl.GLException; import jogamp.opengl.Debug; import com.jogamp.opengl.util.texture.TextureSequence; +import com.jogamp.opengl.util.TimeFrameI; /** * GLMediaPlayer interface specifies a {@link TextureSequence} state machine @@ -116,14 +117,11 @@ import com.jogamp.opengl.util.texture.TextureSequence; * to be properly considered by {@link GLMediaPlayerFactory#create(ClassLoader, String)} * and {@link GLMediaPlayerFactory#createDefault()}. *

+ *
Timestamp Accuracy
 *

- * Variable type, value range and dimension has been chosen to suit embedded CPUs - * and characteristics of audio and video streaming. - * Milliseconds of type integer with a maximum value of {@link Integer#MAX_VALUE} - * will allow tracking time up 2,147,483.647 seconds or - * 24 days 20 hours 31 minutes and 23 seconds. - * Milliseconds granularity is also more than enough to deal with A-V synchronization, - * where the threshold usually lies within 22ms. + *

+ * Timestamp type and value range has been chosen to suit embedded CPUs + * and characteristics of audio and video streaming. See {@link TimeFrameI}. *

* *
Audio and video synchronization
 diff --git a/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java b/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java index e13e5ff13..8b6cc1bf9 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java +++ b/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java @@ -29,6 +29,8 @@ package com.jogamp.opengl.util.texture; import javax.media.opengl.GL; +import com.jogamp.opengl.util.TimeFrameI; + /** * Protocol for texture sequences, like animations, movies, etc. *

@@ -109,35 +111,21 @@ public interface TextureSequence { * Texture holder interface, maybe specialized by implementation * to associated related data. */ - public static class TextureFrame { - /** Constant marking an invalid PTS, i.e. Integer.MIN_VALUE == 0x80000000 == {@value}. Sync w/ native code. */ - public static final int INVALID_PTS = 0x80000000; - - /** Constant marking the end of the stream PTS, i.e. Integer.MIN_VALUE - 1 == 0x7FFFFFFF == {@value}. Sync w/ native code. */ - public static final int END_OF_STREAM_PTS = 0x7FFFFFFF; - + public static class TextureFrame extends TimeFrameI { + public TextureFrame(Texture t, int pts, int duration) { + super(pts, duration); + texture = t; + } public TextureFrame(Texture t) { texture = t; - pts = INVALID_PTS; - duration = 0; } public final Texture getTexture() { return texture; } - /** Get this frame's presentation timestamp (PTS) in milliseconds. */ - public final int getPTS() { return pts; } - /** Set this frame's presentation timestamp (PTS) in milliseconds. */ - public final void setPTS(int pts) { this.pts = pts; } - /** Get this frame's duration in milliseconds. */ - public final int getDuration() { return duration; } - /** Set this frame's duration in milliseconds. */ - public final void setDuration(int duration) { this.duration = duration; } public String toString() { - return "TextureFrame[pts " + pts + " ms, l " + duration + " ms, texID "+ texture.getTextureObject() + "]"; + return "TextureFrame[pts " + pts + " ms, l " + duration + " ms, texID "+ (null != texture ? texture.getTextureObject() : 0) + "]"; } protected final Texture texture; - protected int pts; - protected int duration; } public interface TexSeqEventListener { diff --git a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java index 578a219e9..63d9c8d22 100644 --- a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java +++ b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java @@ -35,6 +35,7 @@ import javax.media.opengl.GLException; import com.jogamp.common.os.AndroidVersion; import com.jogamp.common.os.Platform; +import com.jogamp.opengl.util.TimeFrameI; import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; import com.jogamp.opengl.util.texture.TextureSequence; @@ -227,7 +228,8 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { + protected final int getNextTextureImpl(GL gl, TextureFrame nextFrame) { + int pts = TimeFrameI.INVALID_PTS; if(null != stex && null != mp) { final SurfaceTextureFrame nextSFrame = (SurfaceTextureFrame) nextFrame; final Surface nextSurface = nextSFrame.getSurface(); @@ -254,11 +256,12 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { final SurfaceTexture nextSTex = nextSFrame.getSurfaceTexture(); nextSTex.updateTexImage(); // nextFrame.setPTS( (int) ( nextSTex.getTimestamp() / 1000000L ) ); // nano -9 -> milli -3 - nextFrame.setPTS( mp.getCurrentPosition() ); + pts = mp.getCurrentPosition(); + nextFrame.setPTS( pts ); // stex.getTransformMatrix(atex.getSTMatrix()); } } - return true; + return pts; } @Override diff --git a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java index 8193175b7..238595d45 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java +++ b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java @@ -42,14 +42,17 @@ import javax.media.opengl.GLES2; import javax.media.opengl.GLException; import javax.media.opengl.GLProfile; +import jogamp.opengl.Debug; + import com.jogamp.common.os.Platform; import com.jogamp.common.util.LFRingbuffer; import com.jogamp.common.util.Ringbuffer; +import com.jogamp.opengl.util.TimeFrameI; import com.jogamp.opengl.util.av.AudioSink; -import com.jogamp.opengl.util.av.AudioSink.AudioFrame; import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; import com.jogamp.opengl.util.texture.TextureSequence; +import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame; /** * After object creation an implementation may customize the behavior: @@ -64,6 +67,7 @@ import com.jogamp.opengl.util.texture.TextureSequence; *

*/ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { + private static final int STREAM_WORKER_DELAY = Debug.getIntProperty("jogl.debug.GLMediaPlayer.StreamWorker.delay", false, 0); protected static final String unknown = "unknown"; @@ -151,13 +155,6 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { private ArrayList eventListeners = new ArrayList(); - private static Ringbuffer.AllocEmptyArray rbAllocTextureFrameArray = new Ringbuffer.AllocEmptyArray() { - @Override - public TextureFrame[] newArray(int size) { - return new TextureFrame[size]; - } - }; - protected GLMediaPlayerImpl() { this.textureCount=0; this.textureTarget=GL.GL_TEXTURE_2D; @@ -284,6 +281,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { @Override public final State play() { synchronized( stateLock ) { + final State preState = state; switch( state ) { case Paused: if( playImpl() ) { @@ -296,7 +294,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } default: } - if(DEBUG) { System.err.println("Play: "+toString()); } + if(DEBUG) { System.err.println("Play: "+preState+" -> "+state+", "+toString()); } return state; } } @@ -308,6 +306,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } private final State pauseImpl(int event_mask) { synchronized( stateLock ) { + final State preState = state; if( State.Playing == state ) { event_mask = addStateEventMask(event_mask, GLMediaPlayer.State.Paused); state = State.Paused; @@ -320,7 +319,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { play(); } } - if(DEBUG) { System.err.println("Pause: "+toString()); } + if(DEBUG) { System.err.println("Pause: "+preState+" -> "+state+", "+toString()); } return state; } } @@ -329,6 +328,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { @Override public final int seek(int msec) { synchronized( stateLock ) { + final State preState = state; final int pts1; switch(state) { case Playing: @@ -348,7 +348,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { default: pts1 = 0; } - if(DEBUG) { System.err.println("Seek("+msec+"): "+toString()); } + if(DEBUG) { System.err.println("Seek("+msec+"): "+preState+" -> "+state+", "+toString()); } return pts1; } } @@ -362,6 +362,8 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { @Override public final boolean setPlaySpeed(float rate) { synchronized( stateLock ) { + final State preState = state; + final float preSpeed = playSpeed; boolean res = false; if(State.Uninitialized != state ) { if( rate > 0.01f ) { @@ -371,11 +373,11 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { if( setPlaySpeedImpl(rate) ) { resetAudioVideoPTS(); playSpeed = rate; - if(DEBUG) { System.err.println("SetPlaySpeed: "+toString()); } res = true; } } } + if(DEBUG) { System.err.println("setPlaySpeed("+rate+"): "+preState+"/"+preSpeed+" -> "+state+"/"+playSpeed+", "+toString()); } return res; } } @@ -464,14 +466,16 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { removeAllTextureFrames(gl); initGLImpl(gl); videoFramesOrig = createTexFrames(gl, textureCount); - videoFramesFree = new LFRingbuffer(videoFramesOrig, rbAllocTextureFrameArray); - videoFramesDecoded = new LFRingbuffer(textureCount, rbAllocTextureFrameArray); + videoFramesFree = new LFRingbuffer(videoFramesOrig); + videoFramesDecoded = new LFRingbuffer(TextureFrame[].class, textureCount); lastFrame = videoFramesFree.getBlocking( ); streamWorker.initGL(gl); } else { + removeAllTextureFrames(null); initGLImpl(null); setTextureFormat(-1, -1); setTextureType(-1); + videoFramesOrig = null; videoFramesFree = null; videoFramesDecoded = null; lastFrame = null; @@ -482,6 +486,15 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } } } + /** + * Shall initialize all GL related resources, if not audio-only. + *

+ * Shall also take care of {@link AudioSink} initialization if appropriate. + *

+ * @param gl null for audio-only, otherwise a valid and current GL object. + * @throws IOException + * @throws GLException + */ protected abstract void initGLImpl(GL gl) throws IOException, GLException; /** @@ -570,12 +583,12 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } private final void removeAllTextureFrames(GL gl) { - if( null != videoFramesOrig ) { - final TextureFrame[] texFrames = videoFramesOrig; - videoFramesOrig = null; - videoFramesFree = null; - videoFramesDecoded = null; - lastFrame = null; + final TextureFrame[] texFrames = videoFramesOrig; + videoFramesOrig = null; + videoFramesFree = null; + videoFramesDecoded = null; + lastFrame = null; + if( null != texFrames ) { for(int i=0; i maxVideoDelay ) { cachedFrame = nextFrame; nextFrame = null; - } else if ( dt < -maxVideoDelay ) { + } else if ( !droppedFrame && dt < -maxVideoDelay && videoFramesDecoded.size() > 0 ) { + // only drop if prev. frame has not been dropped and + // frame is too late and one decoded frame is already available. dropFrame = true; } video_pts_last = video_pts; @@ -682,6 +701,20 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { lastFrame = nextFrame; videoFramesFree.putBlocking(_lastFrame); } + } else if( DEBUG ) { + final int video_pts = lastFrame.getPTS(); + final int audio_pts = getAudioPTSImpl(); + final int audio_scr = (int) ( ( currentTimeMillis - audio_scr_t0 ) * playSpeed ); + final int d_apts; + if( audio_pts != TimeFrameI.INVALID_PTS ) { + d_apts = audio_pts - audio_scr; + } else { + d_apts = 0; + } + final int video_scr = video_scr_pts + (int) ( ( currentTimeMillis - video_scr_t0 ) * playSpeed ); + final int d_vpts = video_pts - video_scr; + System.err.println( "AV~: dT "+(currentTimeMillis-lastTimeMillis)+", "+ + getPerfStringImpl( video_scr, video_pts, d_vpts, audio_scr, audio_pts, d_apts, 0 ) + ", droppedFrame "+droppedFrame); } lastTimeMillis = currentTimeMillis; } while( dropFrame ); @@ -696,12 +729,27 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { protected void preNextTextureImpl(GL gl) {} protected void postNextTextureImpl(GL gl) {} /** + * Process stream until the next video frame, i.e. {@link TextureFrame}, has been reached. + * Audio frames, i.e. {@link AudioSink.AudioFrame}, shall be handled in the process. + *

+ * Video frames shall be ignored, if {@link #getVID()} is {@link #STREAM_ID_NONE}. + *

+ *

+ * Audio frames shall be ignored, if {@link #getAID()} is {@link #STREAM_ID_NONE}. + *

+ *

+ * Methods is invoked on the StreamWorker decoding thread. + *

+ *

* Implementation shall care of OpenGL synchronization as required, e.g. glFinish()/glFlush()! - * @param gl - * @param nextFrame - * @return + *

+ * @param gl valid and current GL instance, shall be null for audio only. + * @param nextFrame the {@link TextureFrame} to store the video PTS and texture data, + * shall be null for audio only. + * @return the last processed video PTS value, maybe {@link TimeFrameI#INVALID_PTS} if video frame is invalid or n/a. + * Will be {@link TimeFrameI#END_OF_STREAM_PTS} if end of stream reached. */ - protected abstract boolean getNextTextureImpl(GL gl, TextureFrame nextFrame); + protected abstract int getNextTextureImpl(GL gl, TextureFrame nextFrame); /** * {@inheritDoc} @@ -731,12 +779,12 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { private void flushAllVideoFrames() { if( null != videoFramesFree ) { videoFramesFree.resetFull(videoFramesOrig); + lastFrame = videoFramesFree.get(); + if( null == lastFrame ) { throw new InternalError("XXX"); } } if( null != videoFramesDecoded ) { videoFramesDecoded.clear(); } - lastFrame = videoFramesFree.get( ); - if( null == lastFrame ) { throw new InternalError("XXX"); } cachedFrame = null; } private void resetAllAudioVideoSync() { @@ -842,14 +890,16 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { public synchronized void doPause() { if( isActive ) { shallPause = true; - if( isBlocked && isActive ) { - this.interrupt(); - } - while( isActive ) { - try { - this.wait(); // wait until paused - } catch (InterruptedException e) { - e.printStackTrace(); + if( Thread.currentThread() != this ) { + if( isBlocked && isActive ) { + this.interrupt(); + } + while( isActive ) { + try { + this.wait(); // wait until paused + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } @@ -857,12 +907,14 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { public synchronized void doResume() { if( isRunning && !isActive ) { shallPause = false; - while( !isActive ) { - this.notify(); // wake-up pause-block - try { - this.wait(); // wait until resumed - } catch (InterruptedException e) { - e.printStackTrace(); + if( Thread.currentThread() != this ) { + while( !isActive ) { + this.notify(); // wake-up pause-block + try { + this.wait(); // wait until resumed + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } @@ -870,15 +922,17 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { public synchronized void doStop() { if( isRunning ) { shallStop = true; - if( isBlocked && isRunning ) { - this.interrupt(); - } - while( isRunning ) { - this.notify(); // wake-up pause-block (opt) - try { - this.wait(); // wait until stopped - } catch (InterruptedException e) { - e.printStackTrace(); + if( Thread.currentThread() != this ) { + if( isBlocked && isRunning ) { + this.interrupt(); + } + while( isRunning ) { + this.notify(); // wake-up pause-block (opt) + try { + this.wait(); // wait until stopped + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } @@ -939,51 +993,68 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { preNextTextureImpl(sharedGLCtx.getGL()); sharedGLCtxCurrent = true; } + if( null == videoFramesFree ) { + throw new InternalError("XXX videoFramesFree is null"); + } } } if( !shallStop ) { TextureFrame nextFrame = null; - if( null != sharedGLCtx ) { - try { - isBlocked = true; - nextFrame = videoFramesFree.getBlocking( ); - isBlocked = false; - nextFrame.setPTS( TextureFrame.INVALID_PTS ); // mark invalid until processed! - final GL gl = sharedGLCtxCurrent ? sharedGLCtx.getGL() : null; - if( getNextTextureImpl(gl, nextFrame) ) { + try { + final GL gl; + isBlocked = true; + if( null != videoFramesFree ) { + nextFrame = videoFramesFree.getBlocking(); + nextFrame.setPTS( TimeFrameI.INVALID_PTS ); // mark invalid until processed! + gl = sharedGLCtx.getGL(); + } else { + gl = null; + } + isBlocked = false; + final int vPTS = getNextTextureImpl(gl, nextFrame); + if( TimeFrameI.INVALID_PTS != vPTS ) { + if( null != nextFrame ) { + if( STREAM_WORKER_DELAY > 0 ) { + Thread.sleep(STREAM_WORKER_DELAY); + } if( !videoFramesDecoded.put(nextFrame) ) { throw new InternalError("XXX: free "+videoFramesFree+", decoded "+videoFramesDecoded+", "+GLMediaPlayerImpl.this); } newFrameAvailable(nextFrame, Platform.currentTimeMillis()); nextFrame = null; - } - } catch (InterruptedException e) { - isBlocked = false; - if( !shallStop && !shallPause ) { - streamErr = new StreamException("InterruptedException while decoding: "+GLMediaPlayerImpl.this.toString(), e); - } - } catch (Throwable t) { - streamErr = new StreamException(t.getClass().getSimpleName()+" while decoding: "+GLMediaPlayerImpl.this.toString(), t); - } finally { - if( null != nextFrame ) { // put back - videoFramesFree.put(nextFrame); - } - if( null != streamErr ) { - if( DEBUG ) { - final Throwable t = null != streamErr.getCause() ? streamErr.getCause() : streamErr; - System.err.println("Caught StreamException: "+t.getMessage()); - t.printStackTrace(); + } else { + // audio only + if( TimeFrameI.END_OF_STREAM_PTS == vPTS ) { + // state transition incl. notification + shallPause = true; + isActive = false; + pauseImpl(GLMediaEventListener.EVENT_CHANGE_EOS); } - // state transition incl. notification - shallPause = true; - isActive = false; - pause(); } } - } else { - // audio only - getNextTextureImpl(null, null); + } catch (InterruptedException e) { + isBlocked = false; + if( !shallStop && !shallPause ) { + streamErr = new StreamException("InterruptedException while decoding: "+GLMediaPlayerImpl.this.toString(), e); + } + } catch (Throwable t) { + streamErr = new StreamException(t.getClass().getSimpleName()+" while decoding: "+GLMediaPlayerImpl.this.toString(), t); + } finally { + if( null != nextFrame ) { // put back + videoFramesFree.put(nextFrame); + } + if( null != streamErr ) { + if( DEBUG ) { + final Throwable t = null != streamErr.getCause() ? streamErr.getCause() : streamErr; + System.err.println("Caught StreamException: "+t.getMessage()); + t.printStackTrace(); + } + // state transition incl. notification + shallPause = true; + isActive = false; + pauseImpl(GLMediaEventListener.EVENT_CHANGE_ERR); + } } } } @@ -1106,13 +1177,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { destroyImpl(gl); removeAllTextureFrames(gl); textureCount=0; - if( null != videoFramesFree ) { - videoFramesFree.clear(); - } - if( null != videoFramesDecoded ) { - videoFramesDecoded.clear(); - } - changeState(0, State.Uninitialized); + changeState(0, State.Uninitialized); return state; } } @@ -1215,7 +1280,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { final String audioSinkInfo; final AudioSink audioSink = getAudioSink(); if( null != audioSink ) { - audioSinkInfo = "AudioSink[frames [d "+audioSink.getEnqueuedFrameCount()+", q "+audioSink.getQueuedFrameCount()+", f "+audioSink.getFreeFrameCount()+"], time "+audioSink.getQueuedTime()+", bytes "+audioSink.getQueuedByteCount()+"]"; + audioSinkInfo = "AudioSink[frames [p "+audioSink.getEnqueuedFrameCount()+", q "+audioSink.getQueuedFrameCount()+", f "+audioSink.getFreeFrameCount()+", c "+audioSink.getFrameCount()+"], time "+audioSink.getQueuedTime()+", bytes "+audioSink.getQueuedByteCount()+"]"; } else { audioSinkInfo = ""; } diff --git a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java index 31ac55ec3..9066f3dd1 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java @@ -83,9 +83,10 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { - nextFrame.setPTS( getAudioPTSImpl() ); - return true; + protected final int getNextTextureImpl(GL gl, TextureFrame nextFrame) { + final int pts = getAudioPTSImpl(); + nextFrame.setPTS( pts ); + return pts; } @Override @@ -102,40 +103,42 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { texData = null; } } - - @Override - protected final void initStreamImpl(int vid, int aid) throws IOException { + + public final static TextureData createTestTextureData() { + TextureData res = null; try { - URLConnection urlConn = IOUtil.getResource("jogl/util/data/av/test-ntsc01-160x90.png", this.getClass().getClassLoader()); + URLConnection urlConn = IOUtil.getResource("jogl/util/data/av/test-ntsc01-160x90.png", NullGLMediaPlayer.class.getClassLoader()); if(null != urlConn) { - texData = TextureIO.newTextureData(GLProfile.getGL2ES2(), urlConn.getInputStream(), false, TextureIO.PNG); + res = TextureIO.newTextureData(GLProfile.getGL2ES2(), urlConn.getInputStream(), false, TextureIO.PNG); } } catch (Exception e) { e.printStackTrace(); } - final int _w, _h; - if(null != texData) { - _w = texData.getWidth(); - _h = texData.getHeight(); - } else { - _w = 640; - _h = 480; - ByteBuffer buffer = Buffers.newDirectByteBuffer(_w*_h*4); + if(null == res) { + final int w = 160; + final int h = 90; + ByteBuffer buffer = Buffers.newDirectByteBuffer(w*h*4); while(buffer.hasRemaining()) { buffer.put((byte) 0xEA); buffer.put((byte) 0xEA); buffer.put((byte) 0xEA); buffer.put((byte) 0xEA); } buffer.rewind(); - texData = new TextureData(GLProfile.getGL2ES2(), - GL.GL_RGBA, _w, _h, 0, + res = new TextureData(GLProfile.getGL2ES2(), + GL.GL_RGBA, w, h, 0, GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, false, false, false, buffer, null); } + return res; + } + + @Override + protected final void initStreamImpl(int vid, int aid) throws IOException { + texData = createTestTextureData(); final int r_aid = GLMediaPlayer.STREAM_ID_NONE == aid ? GLMediaPlayer.STREAM_ID_NONE : GLMediaPlayer.STREAM_ID_AUTO; final float _fps = 24f; final int _duration = 10*60*1000; // msec final int _totalFrames = (int) ( (_duration/1000)*_fps ); updateAttributes(GLMediaPlayer.STREAM_ID_AUTO, r_aid, - _w, _h, 0, + texData.getWidth(), texData.getHeight(), 0, 0, 0, _fps, _totalFrames, 0, _duration, "png-static", null); } diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index f1213d751..a800f2a31 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -40,6 +40,7 @@ import javax.media.opengl.GLException; import com.jogamp.common.util.VersionNumber; import com.jogamp.gluegen.runtime.ProcAddressTable; +import com.jogamp.opengl.util.TimeFrameI; import com.jogamp.opengl.util.GLPixelStorageModes; import com.jogamp.opengl.util.av.AudioSink; import com.jogamp.opengl.util.av.AudioSink.AudioDataFormat; @@ -169,7 +170,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { /** Initial audio frame count, ALAudioSink may grow buffer! */ private int initialAudioFrameCount = AV_DEFAULT_AFRAMES; private final int audioFrameGrowAmount = 8; - private final int audioFrameLimit = 128; + private final int audioFrameLimitWithVideo = 64; // 128; + private final int audioFrameLimitAudioOnly = 32; // 64; private SampleFormat aSampleFmt = null; private AudioSink.AudioDataFormat avChosenAudioFormat; private AudioSink.AudioDataFormat sinkChosenAudioFormat; @@ -235,6 +237,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if(null == audioSink) { throw new GLException("AudioSink null"); } + final int audioFrameLimit; if( null != gl ) { final GLContextImpl ctx = (GLContextImpl)gl.getContext(); AccessController.doPrivileged(new PrivilegedAction() { @@ -247,7 +250,10 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { setGLFuncs0(moviePtr, procAddrGLTexSubImage2D, procAddrGLGetError, procAddrGLFlush, procAddrGLFinish); return null; } } ); - } + audioFrameLimit = audioFrameLimitWithVideo; + } else { + audioFrameLimit = audioFrameLimitAudioOnly; + } sinkChosenAudioFormat = audioSink.initSink(avChosenAudioFormat, initialAudioFrameCount, audioFrameGrowAmount, audioFrameLimit); if(DEBUG) { @@ -303,7 +309,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { */ private void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane, int lSz0, int lSz1, int lSz2, - int tWd0, int tWd1, int tWd2, + int tWd0, int tWd1, int tWd2, int tH, int audioFrameCount, int sampleFmt, int sampleRate, int channels) { vPixelFmt = PixelFormat.valueOf(pixFmt); vPlanes = planes; @@ -320,7 +326,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { // w*h + 2 ( w/2 * h/2 ) // w*h + w*h/2 // 2*w/2 * h - texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = height; + texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = tH; break; // case PIX_FMT_YUYV422: case RGB24: @@ -329,7 +335,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { case RGBA: case ABGR: case BGRA: - texWidth = vTexWidth[0]; texHeight = height; + texWidth = vTexWidth[0]; texHeight = tH; break; default: // FIXME: Add more formats ! throw new RuntimeException("Unsupported pixelformat: "+vPixelFmt); @@ -487,42 +493,33 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { + protected final int getNextTextureImpl(GL gl, TextureFrame nextFrame) { if(0==moviePtr) { throw new GLException("FFMPEG native instance null"); } - int vPTS = TextureFrame.INVALID_PTS; - if( null != nextFrame ) { + int vPTS = TimeFrameI.INVALID_PTS; + if( null != gl ) { final Texture tex = nextFrame.getTexture(); tex.enable(gl); tex.bind(gl); } /** Try decode up to 10 packets to find one containing video. */ - for(int i=0; TextureFrame.INVALID_PTS == vPTS && 10 > i; i++) { + for(int i=0; TimeFrameI.INVALID_PTS == vPTS && 10 > i; i++) { vPTS = readNextPacket0(moviePtr, textureTarget, textureFormat, textureType); } - if( TextureFrame.INVALID_PTS != vPTS ) { - if( null != nextFrame ) { - nextFrame.setPTS(vPTS); - } - return true; - } else { - return false; + if( null != nextFrame ) { + nextFrame.setPTS(vPTS); } - } + return vPTS; + } private final void pushSound(ByteBuffer sampleData, int data_size, int audio_pts) { setFirstAudioPTS2SCR( audio_pts ); if( 1.0f == playSpeed || audioSinkPlaySpeedSet ) { - audioSink.enqueueData( new AudioSink.AudioFrame(sampleData, data_size, audio_pts ) ); + audioSink.enqueueData( audio_pts, sampleData, data_size); } } - - private final int getBytesPerMS(int time) { - final int bytesPerSample = sinkChosenAudioFormat.sampleSize >>> 3; // /8 - return time * ( sinkChosenAudioFormat.channelCount * bytesPerSample * ( sinkChosenAudioFormat.sampleRate / 1000 ) ); - } private static native int getAvUtilVersion0(); private static native int getAvFormatVersion0(); diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java index e1b773e9b..faa6a56c4 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java @@ -164,12 +164,13 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected boolean getNextTextureImpl(GL gl, TextureFrame nextFrame) { + protected int getNextTextureImpl(GL gl, TextureFrame nextFrame) { if(0==moviePtr) { throw new GLException("OMX native instance null"); } final int nextTex = _getNextTextureID(moviePtr, true); if(0 < nextTex) { + // FIXME set pts ! /* FIXME final TextureSequence.TextureFrame eglImgTex = texFrameMap.get(new Integer(_getNextTextureID(moviePtr, blocking))); @@ -177,7 +178,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { lastTex = eglImgTex; } */ } - return true; + return 0; // FIXME: return pts } private String replaceAll(String orig, String search, String repl) { -- cgit v1.2.3