diff options
Diffstat (limited to 'src/jogl/classes')
9 files changed, 821 insertions, 414 deletions
diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java index a36bce305..fae88ea18 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java @@ -34,19 +34,37 @@ import javax.media.opengl.GL; import javax.media.opengl.GLException; import jogamp.opengl.Debug; +import jogamp.opengl.util.av.GLMediaPlayerImpl; import com.jogamp.opengl.util.texture.TextureSequence; /** - * Lifecycle of an GLMediaPlayer: + * GLMediaPlayer interface specifies a {@link TextureSequence} + * with a video stream as it's source. + * <p> + * Audio maybe supported and played back internally or via an {@link AudioSink} implementation, + * if an audio stream is selected in {@link #initGLStream(GL, int, URLConnection, int, int)}. + * </p> + * <a name="lifecycle"><h5>GLMediaPlayer Lifecycle</h5></a> + * <p> + * <table border="1"> + * <tr><th>action</th> <th>state before</th> <th>state after</th></tr> + * <tr><td>{@link #initGLStream(GL, int, URLConnection, int, int)}</td> <td>Uninitialized</td> <td>Paused</td></tr> + * <tr><td>{@link #play()}</td> <td>Paused</td> <td>Playing</td></tr> + * <tr><td>{@link #pause()}</td> <td>Playing</td> <td>Paused</td></tr> + * <tr><td>{@link #seek(int)}</td> <td>Playing, Paused</td> <td>Unchanged</td></tr> + * <tr><td>{@link #destroy(GL)}</td> <td>ANY</td> <td>Uninitialized</td></tr> + * </table> + * </p> + * <a name="streamIDs"><h5>Audio and video Stream IDs</h5></a> + * <p> * <table border="1"> - * <tr><th>action</th> <th>state before</th> <th>state after</th></tr> - * <tr><td>{@link #initGLStream(GL, int, URLConnection)}</td> <td>Uninitialized</td> <td>Stopped</td></tr> - * <tr><td>{@link #start()}</td> <td>Stopped, Paused</td> <td>Playing</td></tr> - * <tr><td>{@link #stop()}</td> <td>Playing, Paused</td> <td>Stopped</td></tr> - * <tr><td>{@link #pause()}</td> <td>Playing</td> <td>Paused</td></tr> - * <tr><td>{@link #destroy(GL)}</td> <td>ANY</td> <td>Uninitialized</td></tr> + * <tr><th>value</th> <th>request</th> <th>get</th></tr> + * <tr><td>{@link #STREAM_ID_NONE}</td> <td>mute</td> <td>not available</td></tr> + * <tr><td>{@link #STREAM_ID_AUTO}</td> <td>auto</td> <td>unspecified</td></tr> + * <tr><td>≥0</td> <td>specific stream</td> <td>specific stream</td></tr> * </table> + * </p> * <p> * Current implementations (check each API doc link for details): * <ul> @@ -76,14 +94,21 @@ import com.jogamp.opengl.util.texture.TextureSequence; */ public interface GLMediaPlayer extends TextureSequence { public static final boolean DEBUG = Debug.debug("GLMediaPlayer"); + + /** Constant {@value} for <i>mute</i> or <i>not available</i>. See <a href="#streamIDs">Audio and video Stream IDs</a>. */ + public static final int STREAM_ID_NONE = -2; + /** Constant {@value} for <i>auto</i> or <i>unspecified</i>. See <a href="#streamIDs">Audio and video Stream IDs</a>. */ + public static final int STREAM_ID_AUTO = -1; public interface GLMediaEventListener extends TexSeqEventListener<GLMediaPlayer> { - static final int EVENT_CHANGE_SIZE = 1<<0; - static final int EVENT_CHANGE_FPS = 1<<1; - static final int EVENT_CHANGE_BPS = 1<<2; - static final int EVENT_CHANGE_LENGTH = 1<<3; - static final int EVENT_CHANGE_CODEC = 1<<3; + static final int EVENT_CHANGE_VID = 1<<0; + static final int EVENT_CHANGE_AID = 1<<1; + static final int EVENT_CHANGE_SIZE = 1<<2; + static final int EVENT_CHANGE_FPS = 1<<3; + static final int EVENT_CHANGE_BPS = 1<<4; + static final int EVENT_CHANGE_LENGTH = 1<<5; + static final int EVENT_CHANGE_CODEC = 1<<6; /** * @param mp the event source @@ -93,8 +118,11 @@ public interface GLMediaPlayer extends TextureSequence { public void attributesChanges(GLMediaPlayer mp, int event_mask, long when); } + /** + * See <a href="#lifecycle">GLMediaPlayer Lifecycle</a>. + */ public enum State { - Uninitialized(0), Stopped(1), Playing(2), Paused(3); + Uninitialized(0), Playing(1), Paused(2); public final int id; @@ -120,75 +148,107 @@ public interface GLMediaPlayer extends TextureSequence { * Sets the stream to be used. Initializes all stream related states inclusive OpenGL ones, * if <code>gl</code> is not null. * <p> - * Uninitialized -> Stopped + * <a href="#lifecycle">GLMediaPlayer Lifecycle</a>: Uninitialized -> Paused * </p> * @param gl current GL object. If null, no video output and textures will be available. * @param textureCount desired number of buffered textures to be decoded off-thread, use <code>1</code> for on-thread decoding. * @param urlConn the stream connection + * @param vid video stream id, see <a href="#streamIDs">audio and video Stream IDs</a> + * @param aid video stream id, see <a href="#streamIDs">audio and video Stream IDs</a> * @return the new state * * @throws IllegalStateException if not invoked in state Uninitialized * @throws IOException in case of difficulties to open or process the stream * @throws GLException in case of difficulties to initialize the GL resources */ - public State initGLStream(GL gl, int textureCount, URLConnection urlConn) throws IllegalStateException, GLException, IOException; + public State initGLStream(GL gl, int textureCount, URLConnection urlConn, int vid, int aid) throws IllegalStateException, GLException, IOException; + + /** + * If implementation uses a {@link AudioSink}, it's instance will be returned. + * <p> + * The {@link AudioSink} instance is available after {@link #initGLStream(GL, int, URLConnection, int, int)}, + * if used by implementation. + * </p> + */ + public AudioSink getAudioSink(); /** * Releases the GL and stream resources. * <p> - * <code>ANY</code> -> Uninitialized + * <a href="#lifecycle">GLMediaPlayer Lifecycle</a>: <code>ANY</code> -> Uninitialized * </p> */ public State destroy(GL gl); - public void setPlaySpeed(float rate); + /** + * Sets the playback speed. + * <p> + * Play speed is set to <i>normal</i>, i.e. <code>1.0f</code> + * if <code> abs(1.0f - rate) < 0.01f</code> to simplify test. + * </p> + * @return true if successful, otherwise false, i.e. due to unsupported value range of implementation. + */ + public boolean setPlaySpeed(float rate); public float getPlaySpeed(); /** - * Stopped/Paused -> Playing + * <a href="#lifecycle">GLMediaPlayer Lifecycle</a>: Paused -> Playing */ - public State start(); + public State play(); /** - * Playing -> Paused + * <a href="#lifecycle">GLMediaPlayer Lifecycle</a>: Playing -> Paused */ public State pause(); /** - * Playing/Paused -> Stopped + * Allowed in state Playing and Paused, otherwise ignored, + * see <a href="#lifecycle">GLMediaPlayer Lifecycle</a>. + * + * @param msec absolute desired time position in milliseconds + * @return time current position in milliseconds, after seeking to the desired position + **/ + public int seek(int msec); + + /** + * See <a href="#lifecycle">GLMediaPlayer Lifecycle</a>. + * @return the current state, either Uninitialized, Playing, Paused */ - public State stop(); + public State getState(); /** - * @return the current state, either Uninitialized, Stopped, Playing, Paused + * Return the video stream id, see <a href="#streamIDs">audio and video Stream IDs</a>. */ - public State getState(); + public int getVID(); /** - * @return current streaming position in milliseconds - **/ - public int getCurrentPosition(); - + * Return the audio stream id, see <a href="#streamIDs">audio and video Stream IDs</a>. + */ + public int getAID(); + + /** + * @return the current decoded frame count since {@link #initGLStream(GL, int, URLConnection, int, int)}. + */ + public int getDecodedFrameCount(); + + /** + * @return the current presented frame count since {@link #initGLStream(GL, int, URLConnection, int, int)}, + * increased by {@link #getNextTexture(GL, boolean)}. + */ + public int getPresentedFrameCount(); + /** - * @return current video PTS in milliseconds of {@link #getLastTexture()} + * @return current video presentation timestamp (PTS) in milliseconds of {@link #getLastTexture()} **/ public int getVideoPTS(); /** - * @return current audio PTS in milliseconds. + * @return current audio presentation timestamp (PTS) in milliseconds. **/ public int getAudioPTS(); /** - * Allowed in state Stopped, Playing and Paused, otherwise ignored. - * - * @param msec absolute desired time position in milliseconds - * @return time current position in milliseconds, after seeking to the desired position - **/ - public int seek(int msec); - - /** * {@inheritDoc} */ @Override @@ -225,7 +285,13 @@ public interface GLMediaPlayer extends TextureSequence { * <i>Warning:</i> Optional information, may not be supported by implementation. * @return the total number of video frames */ - public long getTotalFrames(); + public int getVideoFrames(); + + /** + * <i>Warning:</i> Optional information, may not be supported by implementation. + * @return the total number of audio frames + */ + public int getAudioFrames(); /** * @return total duration of stream in msec. @@ -262,6 +328,8 @@ public interface GLMediaPlayer extends TextureSequence { public String toString(); + public String getPerfString(); + public void addEventListener(GLMediaEventListener l); public void removeEventListener(GLMediaEventListener l); diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayerFactory.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayerFactory.java index 2707dd6d2..c7e1ab5e6 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayerFactory.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayerFactory.java @@ -47,10 +47,13 @@ public class GLMediaPlayerFactory { sink = create(cl, FFMPEGMediaPlayerClazzName); } if( null == sink ) { - sink = new NullGLMediaPlayer(); + sink = createNull(); } return sink; } + public static GLMediaPlayer createNull() { + return new NullGLMediaPlayer(); + } public static GLMediaPlayer create(final ClassLoader cl, String implName) { try { diff --git a/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java b/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java index 3f739b2cc..50801e791 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java +++ b/src/jogl/classes/com/jogamp/opengl/util/texture/TextureSequence.java @@ -110,9 +110,12 @@ public interface TextureSequence { * to associated related data. */ public static class TextureFrame { + /** Constant marking an invalid PTS, i.e. Integer.MIN_VALUE {@value}. */ + public static final int INVALID_PTS = Integer.MIN_VALUE; + public TextureFrame(Texture t) { texture = t; - pts = 0; + pts = INVALID_PTS; } public final Texture getTexture() { return texture; } diff --git a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java index 765cda084..e14642c34 100644 --- a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java +++ b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java @@ -28,13 +28,13 @@ package jogamp.opengl.android.av; import java.io.IOException; -import java.nio.Buffer; import javax.media.opengl.GL; import javax.media.opengl.GLES2; import com.jogamp.common.os.AndroidVersion; import com.jogamp.common.os.Platform; +import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; import com.jogamp.opengl.util.texture.TextureSequence; @@ -100,7 +100,7 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final boolean startImpl() { + protected final boolean playImpl() { if(null != mp) { try { mp.start(); @@ -131,22 +131,6 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final boolean stopImpl() { - if(null != mp) { - wakeUp(false); - try { - mp.stop(); - return true; - } catch (IllegalStateException ise) { - if(DEBUG) { - ise.printStackTrace(); - } - } - } - return false; - } - - @Override protected final int seekImpl(int msec) { if(null != mp) { mp.seekTo(msec); @@ -165,15 +149,19 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final int getCurrentPositionImpl() { return null != mp ? mp.getCurrentPosition() : 0; } - - @Override - protected final int getAudioPTSImpl() { return getCurrentPositionImpl(); } + protected final int getAudioPTSImpl() { return null != mp ? mp.getCurrentPosition() : 0; } @Override protected final void destroyImpl(GL gl) { if(null != mp) { wakeUp(false); + try { + mp.stop(); + } catch (IllegalStateException ise) { + if(DEBUG) { + ise.printStackTrace(); + } + } mp.release(); mp = null; } @@ -198,8 +186,13 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final void initGLStreamImpl(GL gl) throws IOException { + protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException { if(null!=mp && null!=urlConn) { + if( GLMediaPlayer.STREAM_ID_NONE == aid ) { + mp.setVolume(0f, 0f); + // FIXME: Disable audio handling + } // else FIXME: Select aid ! + // Note: Both FIXMEs seem to be n/a via Android's MediaPlayer -> Switch to API level 16 MediaCodec/MediaExtractor .. try { final Uri uri = Uri.parse(urlConn.getURL().toExternalForm()); mp.setDataSource(StaticContext.getContext(), uri); @@ -213,20 +206,18 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { if( null == stex ) { throw new InternalError("XXX"); } - final Surface surf = new Surface(stex); - mp.setSurface(surf); - surf.release(); mp.setSurface(null); try { mp.prepare(); } catch (IOException ioe) { throw new IOException("MediaPlayer failed to process stream <"+urlConn.getURL().toExternalForm()+">: "+ioe.getMessage(), ioe); } + final int r_aid = GLMediaPlayer.STREAM_ID_NONE == aid ? GLMediaPlayer.STREAM_ID_NONE : GLMediaPlayer.STREAM_ID_AUTO; final String icodec = "android"; - updateAttributes(mp.getVideoWidth(), mp.getVideoHeight(), - 0, 0, 0, - 0f, 0, mp.getDuration(), - icodec, icodec); + updateAttributes(GLMediaPlayer.STREAM_ID_AUTO, r_aid, + mp.getVideoWidth(), mp.getVideoHeight(), 0, + 0, 0, 0f, + 0, 0, mp.getDuration(), icodec, icodec); } } @@ -264,8 +255,6 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } return true; } - @Override - protected final void syncFrame2Audio(TextureFrame frame) {} @Override protected final TextureSequence.TextureFrame createTexImage(GL gl, int texName) { diff --git a/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java index 57d5ff625..db2146cdc 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java +++ b/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java @@ -84,7 +84,7 @@ public abstract class EGLMediaPlayerImpl extends GLMediaPlayerImpl { } @Override protected final int validateTextureCount(int desiredTextureCount) { - return desiredTextureCount>1 ? desiredTextureCount : 2; + return desiredTextureCount>2 ? Math.max(4, desiredTextureCount) : 2; } @Override diff --git a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java index bc297dc21..c1cfc0d95 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java +++ b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java @@ -42,6 +42,7 @@ import javax.media.opengl.GLES2; import javax.media.opengl.GLException; import javax.media.opengl.GLProfile; +import com.jogamp.opengl.util.av.AudioSink; import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; import com.jogamp.opengl.util.texture.TextureSequence; @@ -62,7 +63,11 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { protected static final String unknown = "unknown"; - protected State state; + /** Default texture count w/o threading, value {@value}. */ + protected static final int TEXTURE_COUNT_DEFAULT = 2; + + protected volatile State state; + private Object stateLock = new Object(); protected int textureCount; protected int textureTarget; @@ -79,30 +84,72 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { protected volatile float playSpeed = 1.0f; - /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ + protected int vid = GLMediaPlayer.STREAM_ID_AUTO; + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ + protected int aid = GLMediaPlayer.STREAM_ID_AUTO; + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int width = 0; - /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int height = 0; - /** Video fps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Video fps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected float fps = 0; - /** Stream bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + protected int frame_period = 0; + /** Stream bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int bps_stream = 0; - /** Video bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Video bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int bps_video = 0; - /** Audio bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Audio bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int bps_audio = 0; - /** In frames. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ - protected int totalFrames = 0; - /** In ms. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** In frames. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ + protected int videoFrames = 0; + /** In frames. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ + protected int audioFrames = 0; + /** In ms. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected int duration = 0; - /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected String acodec = unknown; - /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */ + /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */ protected String vcodec = unknown; - protected int frameNumber = 0; - protected int currentVideoPTS = 0; + protected volatile int decodedFrameCount = 0; + protected int presentedFrameCount = 0; + protected volatile int video_pts_last = 0; + + /** See {@link #getAudioSink()}. Set by implementation if used from within {@link #initGLStreamImpl(GL, int, int)}! */ + protected AudioSink audioSink = null; + protected boolean audioSinkPlaySpeedSet = false; + + /** System Clock Reference (SCR) of first audio PTS at start time. */ + private long audio_scr_t0 = 0; + private boolean audioSCR_reset = true; + /** System Clock Reference (SCR) of first video frame at start time. */ + private long video_scr_t0 = 0; + /** System Clock Reference (SCR) PTS offset, i.e. first video PTS at start time. */ + private int video_scr_pts = 0; + /** Cumulative video pts diff. */ + private float video_dpts_cum = 0; + /** Cumulative video frames. */ + private int video_dpts_count = 0; + /** Number of min frame count required for video cumulative sync. */ + private static final int VIDEO_DPTS_NUM = 20; + /** Cumulative coefficient, value {@value}. */ + private static final float VIDEO_DPTS_COEFF = 0.7943282f; // (float) Math.exp(Math.log(0.01) / VIDEO_DPTS_NUM); + /** Maximum valid video pts diff. */ + private static final int VIDEO_DPTS_MAX = 5000; // 5s max diff + /** Trigger video PTS reset with given cause as bitfield. */ + private volatile int videoSCR_reset = 0; + + private final boolean isSCRCause(int bit) { return 0 != ( bit & videoSCR_reset); } + /** SCR reset due to: Start, Resume, Seek, .. */ + private static final int SCR_RESET_FORCE = 1 << 0; + /** SCR reset due to: PlaySpeed */ + private static final int SCR_RESET_SPEED = 1 << 1; + + /** Latched video PTS reset, to wait until valid pts after invalidation of cached ones. Currently [1..{@link #VIDEO_DPTS_NUM}] frames. */ + private int videoSCR_reset_latch = 0; + protected SyncedRingbuffer<TextureFrame> videoFramesFree = null; protected SyncedRingbuffer<TextureFrame> videoFramesDecoded = null; protected volatile TextureFrame lastFrame = null; @@ -201,144 +248,205 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } @Override - public final float getPlaySpeed() { - return playSpeed; - } + public final int getDecodedFrameCount() { return decodedFrameCount; } + + @Override + public final int getPresentedFrameCount() { return this.presentedFrameCount; } + + @Override + public final int getVideoPTS() { return video_pts_last; } @Override - public final synchronized void setPlaySpeed(float rate) { - if(State.Uninitialized != state && setPlaySpeedImpl(rate)) { - playSpeed = rate; + public final int getAudioPTS() { + if( State.Uninitialized != state ) { + return getAudioPTSImpl(); } - if(DEBUG) { System.err.println("SetPlaySpeed: "+toString()); } + return 0; } - protected abstract boolean setPlaySpeedImpl(float rate); - - public final State start() { - switch( state ) { - case Stopped: - /** fall-through intended */ - case Paused: - if( startImpl() ) { - resumeFramePusher(); - state = State.Playing; - } - default: + /** Override if not using audioSink! */ + protected int getAudioPTSImpl() { + if( null != audioSink ) { + return audioSink.getPTS(); + } else { + return 0; } - if(DEBUG) { System.err.println("Start: "+toString()); } - return state; } - protected abstract boolean startImpl(); - public final State pause() { - if( State.Playing == state && pauseImpl() ) { - pauseFramePusher(); - state = State.Paused; + public final State getState() { return state; } + + public final State play() { + synchronized( stateLock ) { + switch( state ) { + case Paused: + if( playImpl() ) { + resetAudioVideoSCR(SCR_RESET_FORCE); + resumeFramePusher(); + if( null != audioSink ) { + audioSink.play(); + } + state = State.Playing; + } + default: + } + if(DEBUG) { System.err.println("Start: "+toString()); } + return state; } - if(DEBUG) { System.err.println("Pause: "+toString()); } - return state; } - protected abstract boolean pauseImpl(); + protected abstract boolean playImpl(); - public final State stop() { - switch( state ) { - case Playing: - /** fall-through intended */ - case Paused: - if( stopImpl() ) { + public final State pause() { + synchronized( stateLock ) { + if( State.Playing == state ) { + State _state = state; + state = State.Paused; + if( pauseImpl() ) { + _state = State.Paused; pauseFramePusher(); - state = State.Stopped; + if( null != audioSink ) { + audioSink.pause(); + } } - default: + state = _state; + } + if(DEBUG) { System.err.println("Pause: "+toString()); } + return state; } - if(DEBUG) { System.err.println("Stop: "+toString()); } - return state; } - protected abstract boolean stopImpl(); + protected abstract boolean pauseImpl(); - @Override - public final int getCurrentPosition() { - if( State.Uninitialized != state ) { - return getCurrentPositionImpl(); + public final int seek(int msec) { + synchronized( stateLock ) { + final int pts1; + switch(state) { + case Playing: + case Paused: + final State _state = state; + state = State.Paused; + pauseFramePusher(); + resetAudioVideoSCR(SCR_RESET_FORCE); + pts1 = seekImpl(msec); + if( null != audioSink ) { + audioSink.flush(); + if( State.Playing == _state ) { + audioSink.play(); // cont. w/ new data + } + } + resumeFramePusher(); + state = _state; + break; + default: + pts1 = 0; + } + if(DEBUG) { System.err.println("Seek("+msec+"): "+toString()); } + return pts1; } - return 0; } - protected abstract int getCurrentPositionImpl(); + protected abstract int seekImpl(int msec); @Override - public final int getVideoPTS() { return currentVideoPTS; } + public final float getPlaySpeed() { + return playSpeed; + } @Override - public final int getAudioPTS() { - if( State.Uninitialized != state ) { - return getAudioPTSImpl(); + public final boolean setPlaySpeed(float rate) { + synchronized( stateLock ) { + boolean res = false; + if(State.Uninitialized != state ) { + if( rate > 0.01f ) { + if( Math.abs(1.0f - rate) < 0.01f ) { + rate = 1.0f; + } + if( setPlaySpeedImpl(rate) ) { + resetAudioVideoSCR(SCR_RESET_SPEED); + playSpeed = rate; + if(DEBUG) { System.err.println("SetPlaySpeed: "+toString()); } + res = true; + } + } + } + return res; } - return 0; - } - protected abstract int getAudioPTSImpl(); - - public final int seek(int msec) { - final int pts1; - switch(state) { - case Stopped: - case Playing: - case Paused: - pauseFramePusher(); - pts1 = seekImpl(msec); - currentVideoPTS=pts1; - resumeFramePusher(); - break; - default: - pts1 = 0; + } + /** + * Override if not using AudioSink, or AudioSink's {@link AudioSink#setPlaySpeed(float)} is not sufficient! + * <p> + * AudioSink shall respect <code>!audioSinkPlaySpeedSet</code> to determine data_size + * at {@link AudioSink#enqueueData(com.jogamp.opengl.util.av.AudioSink.AudioFrame)}. + * </p> + */ + protected boolean setPlaySpeedImpl(float rate) { + if( null != audioSink ) { + audioSinkPlaySpeedSet = audioSink.setPlaySpeed(rate); } - if(DEBUG) { System.err.println("Seek("+msec+"): "+toString()); } - return pts1; + // still true, even if audioSink rejects command since we deal w/ video sync + // and AudioSink w/ audioSinkPlaySpeedSet at enqueueData(..). + return true; } - protected abstract int seekImpl(int msec); - - public final State getState() { return state; } - + @Override - public final State initGLStream(GL gl, int reqTextureCount, URLConnection urlConn) throws IllegalStateException, GLException, IOException { - if(State.Uninitialized != state) { - throw new IllegalStateException("Instance not in state "+State.Uninitialized+", but "+state+", "+this); - } - this.urlConn = urlConn; - if (this.urlConn != null) { - try { - if( null != gl ) { - removeAllTextureFrames(gl); - textureCount = validateTextureCount(reqTextureCount); - if( textureCount < 2 ) { - throw new InternalError("Validated texture count < 2: "+textureCount); - } - initGLStreamImpl(gl); // also initializes width, height, .. etc - videoFramesFree = new SyncedRingbuffer<TextureFrame>(createTexFrames(gl, textureCount), true /* full */); - if( 2 < textureCount ) { - videoFramesDecoded = new SyncedRingbuffer<TextureFrame>(new TextureFrame[textureCount], false /* full */); - framePusher = new FramePusher(gl, requiresOffthreadGLCtx()); - framePusher.doStart(); - } else { - videoFramesDecoded = null; + public final State initGLStream(GL gl, int reqTextureCount, URLConnection urlConn, int vid, int aid) throws IllegalStateException, GLException, IOException { + synchronized( stateLock ) { + if(State.Uninitialized != state) { + throw new IllegalStateException("Instance not in state "+State.Uninitialized+", but "+state+", "+this); + } + decodedFrameCount = 0; + presentedFrameCount = 0; + this.urlConn = urlConn; + if (this.urlConn != null) { + try { + if( null != gl ) { + removeAllTextureFrames(gl); + textureCount = validateTextureCount(reqTextureCount); + if( textureCount < TEXTURE_COUNT_DEFAULT ) { + throw new InternalError("Validated texture count < "+TEXTURE_COUNT_DEFAULT+": "+textureCount); + } + initGLStreamImpl(gl, vid, aid); // also initializes width, height, .. etc + videoFramesFree = new SyncedRingbuffer<TextureFrame>(createTexFrames(gl, textureCount), true /* full */); + if( TEXTURE_COUNT_DEFAULT < textureCount ) { + videoFramesDecoded = new SyncedRingbuffer<TextureFrame>(new TextureFrame[textureCount], false /* full */); + framePusher = new FramePusher(gl, requiresOffthreadGLCtx()); + framePusher.doStart(); + } else { + videoFramesDecoded = null; + } + lastFrame = videoFramesFree.getBlocking(false /* clearRef */ ); + state = State.Paused; } - lastFrame = videoFramesFree.getBlocking(false /* clearRef */ ); + return state; + } catch (Throwable t) { + throw new GLException("Error initializing GL resources", t); } - state = State.Stopped; - return state; - } catch (Throwable t) { - throw new GLException("Error initializing GL resources", t); } + return state; } - return state; } + /** + * Implementation shall set the following set of data here + * @see #vid + * @see #aid + * @see #width + * @see #height + * @see #fps + * @see #bps_stream + * @see #videoFrames + * @see #audioFrames + * @see #acodec + * @see #vcodec + */ + protected abstract void initGLStreamImpl(GL gl, int vid, int aid) throws IOException; + /** * Returns the validated number of textures to be handled. * <p> - * Default is always 2 textures, last texture and the decoding texture. + * Default is 2 textures w/o threading, last texture and the decoding texture. + * </p> + * <p> + * > 2 textures is used for threaded decoding, a minimum of 4 textures seems reasonable in this case. * </p> */ protected int validateTextureCount(int desiredTextureCount) { - return 2; + return TEXTURE_COUNT_DEFAULT; } protected boolean requiresOffthreadGLCtx() { return false; } @@ -405,6 +513,18 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { mustFlipVertically); } + protected void destroyTexFrame(GL gl, TextureFrame frame) { + frame.getTexture().destroy(gl); + } + + @Override + public final TextureFrame getLastTexture() throws IllegalStateException { + if(State.Uninitialized == state) { + throw new IllegalStateException("Instance not initialized: "+this); + } + return lastFrame; + } + private final void removeAllTextureFrames(GL gl) { if( null != videoFramesFree ) { final TextureFrame[] texFrames = videoFramesFree.getArray(); @@ -417,79 +537,209 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { destroyTexFrame(gl, frame); texFrames[i] = null; } + System.err.println(Thread.currentThread().getName()+"> Clear TexFrame["+i+"]: "+frame+" -> null"); } } textureCount=0; } - protected void destroyTexFrame(GL gl, TextureFrame frame) { - frame.getTexture().destroy(gl); - } - - /** - * Implementation shall set the following set of data here - * @param gl TODO - * @see #width - * @see #height - * @see #fps - * @see #bps_stream - * @see #totalFrames - * @see #acodec - * @see #vcodec - */ - protected abstract void initGLStreamImpl(GL gl) throws IOException; @Override - public final TextureFrame getLastTexture() throws IllegalStateException { - if(State.Uninitialized == state) { - throw new IllegalStateException("Instance not initialized: "+this); + public final TextureFrame getNextTexture(GL gl, boolean blocking) throws IllegalStateException { + synchronized( stateLock ) { + if(State.Uninitialized == state) { + throw new IllegalStateException("Instance not initialized: "+this); + } + if(State.Playing == state) { + TextureFrame nextFrame = null; + boolean ok = true; + boolean dropFrame = false; + try { + do { + if( TEXTURE_COUNT_DEFAULT < textureCount ) { + nextFrame = videoFramesDecoded.getBlocking(false /* clearRef */ ); + } else { + nextFrame = videoFramesFree.getBlocking(false /* clearRef */ ); + if( getNextTextureImpl(gl, nextFrame, blocking) ) { + newFrameAvailable(nextFrame); + } else { + ok = false; + } + } + if( ok ) { + presentedFrameCount++; + final int video_pts; + if( 0 != videoSCR_reset ) { + if( isSCRCause(SCR_RESET_FORCE) ) { + videoSCR_reset_latch = VIDEO_DPTS_NUM / 2; + resetVideoDPTS(); + resetAllVideoPTS(); + } else { + // SCR_RESET_SPEED + videoSCR_reset_latch = 1; + } + videoSCR_reset = 0; + video_pts = TextureFrame.INVALID_PTS; + } else { + video_pts = nextFrame.getPTS(); + } + if( video_pts != TextureFrame.INVALID_PTS ) { + final int frame_period_last = video_pts - video_pts_last; // rendering loop interrupted ? + if( videoSCR_reset_latch > 0 || frame_period_last > frame_period*10 ) { + if( videoSCR_reset_latch > 0 ) { + videoSCR_reset_latch--; + } + setFirstVideoPTS2SCR( video_pts ); + } + final int scr_pts = video_scr_pts + + (int) ( ( System.currentTimeMillis() - video_scr_t0 ) * playSpeed ); + final int d_vpts = video_pts - scr_pts; + if( -VIDEO_DPTS_MAX > d_vpts || d_vpts > VIDEO_DPTS_MAX ) { + if( DEBUG ) { + System.err.println( getPerfStringImpl( scr_pts, video_pts, d_vpts, 0 ) ); + } + } else { + video_dpts_count++; + video_dpts_cum = d_vpts + VIDEO_DPTS_COEFF * video_dpts_cum; + final int video_dpts_avg_diff = getVideoDPTSAvg(); + if( DEBUG ) { + System.err.println( getPerfStringImpl( scr_pts, video_pts, d_vpts, video_dpts_avg_diff ) ); + } + if( blocking && syncAVRequired() ) { + if( !syncAV( (int) ( video_dpts_avg_diff / playSpeed + 0.5f ) ) ) { + resetVideoDPTS(); + dropFrame = true; + } + } + video_pts_last = video_pts; + } + } + final TextureFrame _lastFrame = lastFrame; + lastFrame = nextFrame; + videoFramesFree.putBlocking(_lastFrame); + } + } while( dropFrame ); + } catch (InterruptedException e) { + ok = false; + e.printStackTrace(); + } finally { + if( !ok && null != nextFrame ) { // put back + videoFramesFree.put(nextFrame); + } + } + } + return lastFrame; } - return lastFrame; } + protected abstract boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking); + protected boolean syncAVRequired() { return false; } + /** + * {@inheritDoc} + * <p> + * Note: All {@link AudioSink} operations are performed from {@link GLMediaPlayerImpl}, + * i.e. {@link #play()}, {@link #pause()}, {@link #seek(int)}, {@link #setPlaySpeed(float)}, {@link #getAudioPTS()}. + * </p> + * <p> + * Implementations using an {@link AudioSink} shall write it's instance to {@link #audioSink} + * from within their {@link #initGLStreamImpl(GL, int, int)} implementation. + * </p> + */ @Override - public final synchronized TextureFrame getNextTexture(GL gl, boolean blocking) throws IllegalStateException { - if(State.Uninitialized == state) { - throw new IllegalStateException("Instance not initialized: "+this); + public final AudioSink getAudioSink() { return audioSink; } + + /** + * To be called from implementation at 1st PTS after start + * w/ current pts value in milliseconds. + * @param audio_scr_t0 + */ + protected void setFirstAudioPTS2SCR(int pts) { + if( audioSCR_reset ) { + audio_scr_t0 = System.currentTimeMillis() - pts; + audioSCR_reset = false; } - if(State.Playing == state) { - TextureFrame nextFrame = null; - boolean ok = true; - try { - if( 2 < textureCount ) { - nextFrame = videoFramesDecoded.getBlocking(false /* clearRef */ ); - } else { - nextFrame = videoFramesFree.getBlocking(false /* clearRef */ ); - if( getNextTextureImpl(gl, nextFrame, blocking) ) { - newFrameAvailable(nextFrame); - } else { - ok = false; - } - } - if( ok ) { - currentVideoPTS = nextFrame.getPTS(); - if( blocking ) { - syncFrame2Audio(nextFrame); - } - final TextureFrame _lastFrame = lastFrame; - lastFrame = nextFrame; - videoFramesFree.putBlocking(_lastFrame); - } - } catch (InterruptedException e) { - ok = false; - e.printStackTrace(); - } finally { - if( !ok && null != nextFrame ) { // put back - videoFramesFree.put(nextFrame); - } + } + private void setFirstVideoPTS2SCR(int pts) { + // video_scr_t0 = System.currentTimeMillis() - pts; + video_scr_t0 = System.currentTimeMillis(); + video_scr_pts = pts; + } + private void resetAllVideoPTS() { + if( null != videoFramesFree ) { + final TextureFrame[] texFrames = videoFramesFree.getArray(); + for(int i=0; i<texFrames.length; i++) { + final TextureFrame frame = texFrames[i]; + frame.setPTS(TextureFrame.INVALID_PTS); + } + } + } + private void resetVideoDPTS() { + video_dpts_cum = 0; + video_dpts_count = 0; + } + private final int getVideoDPTSAvg() { + if( video_dpts_count < VIDEO_DPTS_NUM ) { + return 0; + } else { + return (int) ( video_dpts_cum * (1.0f - VIDEO_DPTS_COEFF) + 0.5f ); + } + } + + private void resetAudioVideoSCR(int cause) { + audioSCR_reset = true; + videoSCR_reset |= cause; + } + + /** + * Synchronizes A-V. + * <p> + * https://en.wikipedia.org/wiki/Audio_to_video_synchronization + * <pre> + * d_av = v_pts - a_pts; + * </pre> + * </p> + * <p> + * Recommendation of audio/video pts time lead/lag at production: + * <ul> + * <li>Overall: +40ms and -60ms audio ahead video / audio after video</li> + * <li>Each stage: +5ms and -15ms. audio ahead video / audio after video</li> + * </ul> + * </p> + * <p> + * Recommendation of av pts time lead/lag at presentation: + * <ul> + * <li>TV: +15ms and -45ms. audio ahead video / audio after video.</li> + * <li>Film: +22ms and -22ms. audio ahead video / audio after video.</li> + * </ul> + * </p> + * <p> + * Maybe implemented as follows: + * <pre> + * d_av = vpts - apts; + * d_av < -22: audio after video == video ahead audio -> drop + * d_av > 22: audio ahead video == video after audio -> sleep(d_av - 10) + * </pre> + * </p> + * <p> + * Returns true if audio is ahead of video, otherwise false (video is ahead of audio). + * In case of the latter (false), the video frame shall be dropped! + * </p> + * @param frame + * @return true if audio is ahead of video, otherwise false (video is ahead of audio) + */ + protected boolean syncAV(int d_vpts) { + if( d_vpts > 22 ) { + if( DEBUG ) { + System.err.println("V (sleep): "+(d_vpts - 22 / 2)+" ms"); } + try { + Thread.sleep( d_vpts - 22 / 2 ); + } catch (InterruptedException e) { } } - return lastFrame; + return true; } - protected abstract boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking); - protected abstract void syncFrame2Audio(TextureFrame frame); private final void newFrameAvailable(TextureFrame frame) { - frameNumber++; + decodedFrameCount++; synchronized(eventListenersLock) { for(Iterator<GLMediaEventListener> i = eventListeners.iterator(); i.hasNext(); ) { i.next().newFrameAvailable(this, frame, System.currentTimeMillis()); @@ -500,6 +750,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { class FramePusher extends Thread { private volatile boolean isRunning = false; private volatile boolean isActive = false; + private volatile boolean isBlocked = false; private volatile boolean shallPause = true; private volatile boolean shallStop = false; @@ -560,6 +811,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { public synchronized void doPause() { if( isActive ) { shallPause = true; + if( isBlocked && isActive ) { + this.interrupt(); + } while( isActive ) { try { this.wait(); // wait until paused @@ -595,6 +849,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { public synchronized void doStop() { if( isRunning ) { shallStop = true; + if( isBlocked && isRunning ) { + this.interrupt(); + } while( isRunning ) { this.notify(); // wake-up pause-block (opt) try { @@ -629,7 +886,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { try { this.wait(); // wait until resumed } catch (InterruptedException e) { - e.printStackTrace(); + if( !shallPause ) { + e.printStackTrace(); + } } } isActive = true; @@ -639,23 +898,30 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { if( !shallStop ) { TextureFrame nextFrame = null; - boolean ok = false; try { - nextFrame = videoFramesFree.getBlocking(true /* clearRef */ ); + isBlocked = true; + nextFrame = videoFramesFree.getBlocking(false /* clearRef */ ); + isBlocked = false; + nextFrame.setPTS( TextureFrame.INVALID_PTS ); // mark invalid until processed! if( getNextTextureImpl(gl, nextFrame, true) ) { - gl.glFinish(); - videoFramesDecoded.putBlocking(nextFrame); - newFrameAvailable(nextFrame); - ok = true; + // gl.glFinish(); + gl.glFlush(); // even better: sync object! + if( !videoFramesDecoded.put(nextFrame) ) { + throw new InternalError("XXX: "+GLMediaPlayerImpl.this); + } + final TextureFrame _nextFrame = nextFrame; + nextFrame = null; + newFrameAvailable(_nextFrame); } } catch (InterruptedException e) { + isBlocked = false; if( !shallStop && !shallPause ) { e.printStackTrace(); // oops shallPause = false; shallStop = true; } } finally { - if( !ok && null != nextFrame ) { // put back + if( null != nextFrame ) { // put back videoFramesFree.put(nextFrame); } } @@ -689,10 +955,18 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } } - protected final void updateAttributes(int width, int height, int bps_stream, int bps_video, int bps_audio, - float fps, int totalFrames, int duration, - String vcodec, String acodec) { + protected final void updateAttributes(int vid, int aid, int width, int height, int bps_stream, + int bps_video, int bps_audio, float fps, + int videoFrames, int audioFrames, int duration, String vcodec, String acodec) { int event_mask = 0; + if( this.vid != vid ) { + event_mask |= GLMediaEventListener.EVENT_CHANGE_VID; + this.vid = vid; + } + if( this.aid != aid ) { + event_mask |= GLMediaEventListener.EVENT_CHANGE_AID; + this.aid = aid; + } if( this.width != width || this.height != height ) { event_mask |= GLMediaEventListener.EVENT_CHANGE_SIZE; this.width = width; @@ -701,6 +975,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { if( this.fps != fps ) { event_mask |= GLMediaEventListener.EVENT_CHANGE_FPS; this.fps = fps; + this.frame_period = (int) ( 1000f / fps + 0.5f ); } if( this.bps_stream != bps_stream || this.bps_video != bps_video || this.bps_audio != bps_audio ) { event_mask |= GLMediaEventListener.EVENT_CHANGE_BPS; @@ -708,9 +983,10 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { this.bps_video = bps_video; this.bps_audio = bps_audio; } - if( this.totalFrames != totalFrames || this.duration != duration ) { + if( this.videoFrames != videoFrames || this.audioFrames != audioFrames || this.duration != duration ) { event_mask |= GLMediaEventListener.EVENT_CHANGE_LENGTH; - this.totalFrames = totalFrames; + this.videoFrames = videoFrames; + this.audioFrames = audioFrames; this.duration = duration; } if( (null!=acodec && acodec.length()>0 && !this.acodec.equals(acodec)) ) { @@ -736,78 +1012,120 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } @Override - public final synchronized State destroy(GL gl) { - destroyFramePusher(); - destroyImpl(gl); - removeAllTextureFrames(gl); - state = State.Uninitialized; - return state; + public final State destroy(GL gl) { + synchronized( stateLock ) { + destroyFramePusher(); + destroyImpl(gl); + removeAllTextureFrames(gl); + state = State.Uninitialized; + return state; + } } protected abstract void destroyImpl(GL gl); @Override - public final synchronized URLConnection getURLConnection() { + public final URLConnection getURLConnection() { return urlConn; } @Override - public final synchronized String getVideoCodec() { + public final int getVID() { return vid; } + + @Override + public final int getAID() { return aid; } + + @Override + public final String getVideoCodec() { return vcodec; } @Override - public final synchronized String getAudioCodec() { + public final String getAudioCodec() { return acodec; } @Override - public final synchronized long getTotalFrames() { - return totalFrames; + public final int getVideoFrames() { + return videoFrames; + } + + public final int getAudioFrames() { + return audioFrames; } @Override - public final synchronized int getDuration() { + public final int getDuration() { return duration; } @Override - public final synchronized long getStreamBitrate() { + public final long getStreamBitrate() { return bps_stream; } @Override - public final synchronized int getVideoBitrate() { + public final int getVideoBitrate() { return bps_video; } @Override - public final synchronized int getAudioBitrate() { + public final int getAudioBitrate() { return bps_audio; } @Override - public final synchronized float getFramerate() { + public final float getFramerate() { return fps; } @Override - public final synchronized int getWidth() { + public final int getWidth() { return width; } @Override - public final synchronized int getHeight() { + public final int getHeight() { return height; } @Override - public final synchronized String toString() { - final float ct = getCurrentPosition() / 1000.0f, tt = getDuration() / 1000.0f; + public final String toString() { + final float tt = getDuration() / 1000.0f; final String loc = ( null != urlConn ) ? urlConn.getURL().toExternalForm() : "<undefined stream>" ; - return "GLMediaPlayer["+state+", "+frameNumber+"/"+totalFrames+" frames, "+ct+"/"+tt+"s, speed "+playSpeed+", "+bps_stream+" bps, "+ - "Texture[count "+textureCount+", target "+toHexString(textureTarget)+", format "+toHexString(textureFormat)+", type "+toHexString(textureType)+"], "+ - "Stream[Video[<"+vcodec+">, "+width+"x"+height+", "+fps+" fps, "+bps_video+" bsp], "+ - "Audio[<"+acodec+">, "+bps_audio+" bsp]], "+loc+"]"; + final int freeVideoFrames = null != videoFramesFree ? videoFramesFree.size() : 0; + final int decVideoFrames = null != videoFramesDecoded ? videoFramesDecoded.size() : 0; + return "GLMediaPlayer["+state+", frames[p "+presentedFrameCount+", d "+decodedFrameCount+", t "+videoFrames+" ("+tt+" s)], "+ + "speed "+playSpeed+", "+bps_stream+" bps, "+ + "Texture[count "+textureCount+", free "+freeVideoFrames+", dec "+decVideoFrames+", target "+toHexString(textureTarget)+", format "+toHexString(textureFormat)+", type "+toHexString(textureType)+"], "+ + "Video[id "+vid+", <"+vcodec+">, "+width+"x"+height+", "+fps+" fps, "+bps_video+" bps], "+ + "Audio[id "+aid+", <"+acodec+">, "+bps_audio+" bps, "+audioFrames+" frames], uri "+loc+"]"; + } + + @Override + public final String getPerfString() { + final int scr_pts = video_scr_pts + + (int) ( ( System.currentTimeMillis() - video_scr_t0 ) * playSpeed ); + final int d_vpts = video_pts_last - scr_pts; + return getPerfStringImpl( scr_pts, video_pts_last, d_vpts, getVideoDPTSAvg() ); + } + private final String getPerfStringImpl(final int scr_pts, final int video_pts, final int d_vpts, final int video_dpts_avg_diff) { + final float tt = getDuration() / 1000.0f; + final int audio_scr = (int) ( ( System.currentTimeMillis() - audio_scr_t0 ) * playSpeed ); + final int audio_pts = getAudioPTSImpl(); + final int d_apts = audio_pts - audio_scr; + final String audioSinkInfo; + final AudioSink audioSink = getAudioSink(); + if( null != audioSink ) { + audioSinkInfo = "AudioSink[frames [d "+audioSink.getEnqueuedFrameCount()+", q "+audioSink.getQueuedFrameCount()+", f "+audioSink.getFreeFrameCount()+"], time "+audioSink.getQueuedTime()+", bytes "+audioSink.getQueuedByteCount()+"]"; + } else { + audioSinkInfo = ""; + } + final int freeVideoFrames = null != videoFramesFree ? videoFramesFree.size() : 0; + final int decVideoFrames = null != videoFramesDecoded ? videoFramesDecoded.size() : 0; + return state+", frames[p "+presentedFrameCount+", d "+decodedFrameCount+", t "+videoFrames+" ("+tt+" s)], "+ + "speed " + playSpeed+", vSCR "+scr_pts+", vpts "+video_pts+", dSCR["+d_vpts+", avrg "+video_dpts_avg_diff+"], "+ + "aSCR "+audio_scr+", apts "+audio_pts+" ( "+d_apts+" ), "+audioSinkInfo+ + ", Texture[count "+textureCount+", free "+freeVideoFrames+", dec "+decVideoFrames+"]"; } @Override @@ -831,7 +1149,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { } @Override - public final synchronized GLMediaEventListener[] getEventListeners() { + public final GLMediaEventListener[] getEventListeners() { synchronized(eventListenersLock) { return eventListeners.toArray(new GLMediaEventListener[eventListeners.size()]); } diff --git a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java index f1ce42257..5d70ca33d 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java @@ -38,6 +38,7 @@ import jogamp.opengl.util.av.GLMediaPlayerImpl; import com.jogamp.common.nio.Buffers; import com.jogamp.common.util.IOUtil; +import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; import com.jogamp.opengl.util.texture.TextureData; import com.jogamp.opengl.util.texture.TextureIO; @@ -62,7 +63,7 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final boolean startImpl() { + protected final boolean playImpl() { pos_start = (int)System.currentTimeMillis(); return true; } @@ -73,11 +74,6 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final boolean stopImpl() { - return true; - } - - @Override protected final int seekImpl(int msec) { pos_ms = msec; validatePos(); @@ -86,20 +82,16 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { @Override protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking) { + nextFrame.setPTS( getAudioPTSImpl() ); return true; } - @Override - protected final void syncFrame2Audio(TextureFrame frame) { } @Override - protected final int getCurrentPositionImpl() { + protected final int getAudioPTSImpl() { pos_ms = (int)System.currentTimeMillis() - pos_start; validatePos(); return pos_ms; } - @Override - protected final int getAudioPTSImpl() { return getCurrentPositionImpl(); } - @Override protected final void destroyImpl(GL gl) { @@ -110,7 +102,7 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final void initGLStreamImpl(GL gl) throws IOException { + protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException { try { URLConnection urlConn = IOUtil.getResource("jogl/util/data/av/test-ntsc01-160x90.png", this.getClass().getClassLoader()); if(null != urlConn) { @@ -136,13 +128,14 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, false, false, false, buffer, null); } + final int r_aid = GLMediaPlayer.STREAM_ID_NONE == aid ? GLMediaPlayer.STREAM_ID_NONE : GLMediaPlayer.STREAM_ID_AUTO; final float _fps = 24f; final int _duration = 10*60*1000; // msec final int _totalFrames = (int) ( (_duration/1000)*_fps ); - updateAttributes(_w, _h, - 0, 0, 0, - _fps, _totalFrames, _duration, - "png-static", null); + updateAttributes(GLMediaPlayer.STREAM_ID_AUTO, r_aid, + _w, _h, 0, + 0, 0, _fps, + _totalFrames, 0, _duration, "png-static", null); } @Override diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index 83a5960f1..dc7ceae39 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -42,9 +42,11 @@ import com.jogamp.common.util.VersionNumber; import com.jogamp.gluegen.runtime.ProcAddressTable; import com.jogamp.opengl.util.GLPixelStorageModes; import com.jogamp.opengl.util.av.AudioSink; +import com.jogamp.opengl.util.av.AudioSink.AudioDataFormat; +import com.jogamp.opengl.util.av.AudioSink.AudioDataType; import com.jogamp.opengl.util.av.AudioSinkFactory; +import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; -import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame; import jogamp.opengl.GLContextImpl; import jogamp.opengl.util.av.GLMediaPlayerImpl; @@ -136,13 +138,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { ( vers >> 8 ) & 0xFF, ( vers >> 0 ) & 0xFF ); } + + // + // General + // + + protected long moviePtr = 0; // // Video // - protected long moviePtr = 0; - protected GLPixelStorageModes psm; protected PixelFormat vPixelFmt = null; protected int vPlanes = 0; protected int vBitsPerPixel = 0; @@ -152,15 +158,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { protected int texWidth, texHeight; // overall (stuffing planes in one texture) protected ByteBuffer texCopy; protected String singleTexComp = "r"; + protected GLPixelStorageModes psm; // // Audio // - protected final int AudioFrameCount = 8; - protected final AudioSink audioSink; - protected final int maxAvailableAudio; - protected AudioSink.AudioDataFormat chosenAudioFormat; + protected static final int AFRAMES_PER_VFRAME = 8; + protected int aFrameCount = 0; + protected SampleFormat aSampleFmt = null; + protected AudioSink.AudioDataFormat avChosenAudioFormat; + protected AudioSink.AudioDataFormat sinkChosenAudioFormat; public FFMPEGMediaPlayer() { if(!available) { @@ -171,12 +179,11 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { throw new GLException("Couldn't create FFMPEGInstance"); } psm = new GLPixelStorageModes(); - audioSink = AudioSinkFactory.createDefault(); - maxAvailableAudio = audioSink.getQueuedByteCount(); + audioSink = null; } @Override protected final int validateTextureCount(int desiredTextureCount) { - return desiredTextureCount>1 ? desiredTextureCount : 2; + return desiredTextureCount>2 ? Math.max(4, desiredTextureCount) : 2; } @Override protected final boolean requiresOffthreadGLCtx() { return true; } @@ -187,10 +194,18 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { destroyInstance0(moviePtr); moviePtr = 0; } + destroyAudioSink(); + } + private final void destroyAudioSink() { + final AudioSink _audioSink = audioSink; + if( null != _audioSink ) { + audioSink = null; + _audioSink.destroy(); + } } @Override - protected final void initGLStreamImpl(GL gl) throws IOException { + protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException { if(0==moviePtr) { throw new GLException("FFMPEG native instance null"); } @@ -209,11 +224,32 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } final String urlS=urlConn.getURL().toExternalForm(); + + aFrameCount = AFRAMES_PER_VFRAME * textureCount + AFRAMES_PER_VFRAME/2; - chosenAudioFormat = audioSink.initSink(audioSink.getPreferredFormat(), AudioFrameCount); System.err.println("setURL: p1 "+this); - setStream0(moviePtr, urlS, -1, -1, AudioFrameCount); // issues updateAttributes*(..) - System.err.println("setURL: p2 "+this); + destroyAudioSink(); + AudioSink _audioSink; + if( GLMediaPlayer.STREAM_ID_NONE == aid ) { + _audioSink = AudioSinkFactory.createNull(); + } else { + _audioSink = AudioSinkFactory.createDefault(); + } + final AudioDataFormat preferredAudioFormat = _audioSink.getPreferredFormat(); + // setStream(..) issues updateAttributes*(..), and defines avChosenAudioFormat, vid, aid, .. etc + setStream0(moviePtr, urlS, vid, aid, aFrameCount, preferredAudioFormat.channelCount, preferredAudioFormat.sampleRate); + // final int audioBytesPerFrame = bps_audio/8000 * frame_period * textureCount; + + System.err.println("setURL: p2 preferred "+preferredAudioFormat+", avChosen "+avChosenAudioFormat+", "+this); + sinkChosenAudioFormat = _audioSink.initSink(avChosenAudioFormat, aFrameCount); + System.err.println("setURL: p3 avChosen "+avChosenAudioFormat+", chosen "+sinkChosenAudioFormat); + if( null == sinkChosenAudioFormat ) { + System.err.println("AudioSink "+_audioSink.getClass().getName()+" does not support "+avChosenAudioFormat+", using Null"); + _audioSink.destroy(); + _audioSink = AudioSinkFactory.createNull(); + sinkChosenAudioFormat = _audioSink.initSink(avChosenAudioFormat, aFrameCount); + } + audioSink = _audioSink; int tf, tif=GL.GL_RGBA; // texture format and internal format switch(vBytesPerPixelPerPlane) { @@ -256,7 +292,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { private void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane, int lSz0, int lSz1, int lSz2, - int tWd0, int tWd1, int tWd2) { + int tWd0, int tWd1, int tWd2, + int sampleFmt, int sampleRate, int channels) { vPixelFmt = PixelFormat.valueOf(pixFmt); vPlanes = planes; vBitsPerPixel = bitsPerPixel; @@ -286,12 +323,53 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { default: // FIXME: Add more planar formats ! throw new RuntimeException("Unsupported pixelformat: "+vPixelFmt); } + + aSampleFmt = SampleFormat.valueOf(sampleFmt); + final int sampleSize; + final boolean signed, fixedP; + switch( aSampleFmt ) { + case S32: + case S32P: + sampleSize = 32; + signed = true; + fixedP = true; + break; + case S16: + case S16P: + sampleSize = 16; + signed = true; + fixedP = true; + break; + case U8: + case U8P: + sampleSize = 8; + signed = false; + fixedP = true; + break; + case DBL: + case DBLP: + sampleSize = 64; + signed = true; + fixedP = true; + break; + case FLT: + case FLTP: + sampleSize = 32; + signed = true; + fixedP = true; + break; + default: // FIXME: Add more planar formats ! + throw new RuntimeException("Unsupported sampleformat: "+aSampleFmt); + } + avChosenAudioFormat = new AudioDataFormat(AudioDataType.PCM, sampleRate, sampleSize, channels, signed, fixedP, true /* littleEndian */); + if(DEBUG) { - System.err.println("XXX0: fmt "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane); + System.err.println("audio: fmt "+aSampleFmt+", "+avChosenAudioFormat); + System.err.println("video: fmt "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane); for(int i=0; i<3; i++) { - System.err.println("XXX0 "+i+": "+vTexWidth[i]+"/"+vLinesize[i]); + System.err.println("video: "+i+": "+vTexWidth[i]+"/"+vLinesize[i]); } - System.err.println("XXX0 total tex "+texWidth+"x"+texHeight); + System.err.println("video: total tex "+texWidth+"x"+texHeight); } } @@ -355,54 +433,27 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } @Override - protected final synchronized int getCurrentPositionImpl() { - return 0!=moviePtr ? getVideoPTS0(moviePtr) : 0; - } - - @Override - public final int getAudioPTSImpl() { return 0; } - - @Override - protected final synchronized boolean setPlaySpeedImpl(float rate) { - return true; - } - - @Override - public final synchronized boolean startImpl() { + public final boolean playImpl() { if(0==moviePtr) { return false; } return true; } - /** @return time position after issuing the command */ @Override - public final synchronized boolean pauseImpl() { + public final boolean pauseImpl() { if(0==moviePtr) { return false; } return true; } - /** @return time position after issuing the command */ - @Override - public final synchronized boolean stopImpl() { - if(0==moviePtr) { - return false; - } - return true; - } - - /** @return time position after issuing the command */ @Override protected final synchronized int seekImpl(int msec) { if(0==moviePtr) { throw new GLException("FFMPEG native instance null"); } - int pts0 = getVideoPTS0(moviePtr); - int pts1 = seek0(moviePtr, msec); - System.err.println("Seek: "+pts0+" -> "+msec+" : "+pts1); - return pts1; + return seek0(moviePtr, msec); } @Override @@ -427,7 +478,6 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { psm.restore(gl); } if( 0 < avPTS ) { - vSTS = avPTS; nextFrame.setPTS(avPTS); return true; } else { @@ -436,50 +486,15 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } private final void pushSound(ByteBuffer sampleData, int data_size, int audio_pts) { - aSTS = audio_pts; - final AudioSink.AudioFrame frame = new AudioSink.AudioFrame(sampleData, data_size, audio_pts); - if( audioSink.isDataAvailable(frame.dataSize) ) { - audioSink.writeData(frame); + setFirstAudioPTS2SCR( audio_pts ); + if( 1.0f == playSpeed || audioSinkPlaySpeedSet ) { + audioSink.enqueueData( new AudioSink.AudioFrame(sampleData, data_size, audio_pts ) ); } } - - /** last audio streaming TS */ - private int aSTS = 0; - /** last video streaming TS */ - private int vSTS = 0; - - private long lastAudioTime = 0; - private static final int audio_dt_d = 400; - private long lastVideoTime = 0; - private static final int video_dt_d = 9; - + @Override - protected final void syncFrame2Audio(TextureFrame frame) { - /** - // poor mans video sync .. TODO: off thread 'readNextPackage0(..)' on shared GLContext and multi textures/unit! - final long now = System.currentTimeMillis(); - // Try sync video to audio - final long now_d = now - lastAudioTime; - final long pts_d = vSTS - aSTS - 444; // hack 444 == play video 444ms ahead of audio - final long dt = Math.min(47, (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ) ; - //final long dt = (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ; - final boolean sleep = dt>video_dt_d && dt<1000 && audioSink.getQueuedByteCount()<maxAvailableAudio-10000; - final long sleepP = dt-video_dt_d; - if(DEBUG) { - final int qAT = audioSink.getQueuedTime(); - System.err.println("s: pts-v "+vSTS+", qAT "+qAT+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt+", sleep "+sleep+", sleepP "+sleepP+" ms"); - } - // ?? Maybe use audioSink.getQueuedTime(); - if( sleep ) { - try { - Thread.sleep(sleepP); - } catch (InterruptedException e) { } - lastVideoTime = System.currentTimeMillis(); - } else { - lastVideoTime = now; - } - */ - } + protected final boolean syncAVRequired() { return true; } + private static native int getAvUtilVersion0(); private static native int getAvFormatVersion0(); private static native int getAvCodecVersion0(); @@ -488,10 +503,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { private native void destroyInstance0(long moviePtr); /** - * Issues {@link #updateAttributes(int, int, int, int, int, float, int, int, String, String)} + * Issues {@link #updateAttributes(int, int, int, int, int, int, int, float, int, int, String, String)} * and {@link #updateAttributes2(int, int, int, int, int, int, int, int, int, int)}. + * <p> + * Always uses {@link AudioSink.AudioDataFormat}: + * <pre> + * [type PCM, sampleRate [10000(?)..44100..48000], sampleSize 16, channelCount 1-2, signed, littleEndian] + * </pre> + * </p> */ - private native void setStream0(long moviePtr, String url, int vid, int aid, int audioFrameCount); + private native void setStream0(long moviePtr, String url, int vid, int aid, int audioFrameCount, + int aChannelCount, int aSampleRate); private native void setGLFuncs0(long moviePtr, long procAddrGLTexSubImage2D, long procAddrGLGetError); private native int getVideoPTS0(long moviePtr); @@ -505,6 +527,32 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { private native int readNextPacket0(long moviePtr, int texTarget, int texFmt, int texType); private native int seek0(long moviePtr, int position); + + public static enum SampleFormat { + // NONE = -1, + U8, ///< unsigned 8 bits + S16, ///< signed 16 bits + S32, ///< signed 32 bits + FLT, ///< float + DBL, ///< double + + U8P, ///< unsigned 8 bits, planar + S16P, ///< signed 16 bits, planar + S32P, ///< signed 32 bits, planar + FLTP, ///< float, planar + DBLP, ///< double, planar + + COUNT; ///< Number of sample formats. + + public static SampleFormat valueOf(int i) { + for (SampleFormat fmt : SampleFormat.values()) { + if(fmt.ordinal() == i) { + return fmt; + } + } + return null; + } + }; public static enum PixelFormat { // NONE= -1, diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java index a21bb40a8..d03cad28a 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java @@ -88,15 +88,16 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { @Override protected void destroyImpl(GL gl) { - _detachVideoRenderer(moviePtr); if (moviePtr != 0) { + _stop(moviePtr); + _detachVideoRenderer(moviePtr); _destroyInstance(moviePtr); moviePtr = 0; } } @Override - protected void initGLStreamImpl(GL gl) throws IOException { + protected void initGLStreamImpl(GL gl, int vid, int aid) throws IOException { if(0==moviePtr) { throw new GLException("OMX native instance null"); } @@ -113,12 +114,8 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - protected int getCurrentPositionImpl() { - return 0!=moviePtr ? _getCurrentPosition(moviePtr) : 0; - } - @Override protected int getAudioPTSImpl() { - return getCurrentPositionImpl(); + return 0!=moviePtr ? _getCurrentPosition(moviePtr) : 0; } @Override @@ -131,7 +128,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } @Override - public synchronized boolean startImpl() { + public synchronized boolean playImpl() { if(0==moviePtr) { return false; } @@ -151,16 +148,6 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { /** @return time position after issuing the command */ @Override - public synchronized boolean stopImpl() { - if(0==moviePtr) { - return false; - } - _stop(moviePtr); - return true; - } - - /** @return time position after issuing the command */ - @Override protected int seekImpl(int msec) { if(0==moviePtr) { throw new GLException("OMX native instance null"); @@ -184,8 +171,6 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl { } return true; } - @Override - protected void syncFrame2Audio(TextureFrame frame) { } private String replaceAll(String orig, String search, String repl) { String dest=null; |