aboutsummaryrefslogtreecommitdiffstats
path: root/src/jogl/classes/jogamp/opengl
diff options
context:
space:
mode:
authorSven Gothel <[email protected]>2013-08-14 07:02:59 +0200
committerSven Gothel <[email protected]>2013-08-14 07:02:59 +0200
commitc37629ea8fdcb11f7f8a18e37a4cde57d4ba6a01 (patch)
tree96e6ef2b8db44f3dd331ac78a0a52d5d5ea15e50 /src/jogl/classes/jogamp/opengl
parentbc3776633ccad81199a96ff8116195133d862395 (diff)
GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-3) - WIP
- GLMediaPlayer - Remove State.Stopped and method stop() - redundant, use pause() / destroy() - Add notion of stream IDs - Add API doc: State / Stream-ID incl. html-anchor - Expose video/audio PTS, .. - Expose optional AudioSink - Min multithreaded textureCount is 4 (EGL* and FFMPEG*) - GLMediaPlayerImpl - Move AudioSink rel. impl. to this class, allowing a tight video implementation reusing logic. - Remove 'synchronized' methods, synchronize on State where applicable - implement new methods (see above) - playSpeed is handled partially in AudioSink. If it exeeds AudioSink's capabilities, drop audio and rely solely on video sync. - video sync (WIP) - video pts delay based on geometric weight - reset video SCR if 'out of range', resync w/ PTS - - FramePusher - allow interruption when pausing/stopping, while waiting for next avail free frame to decode. - FFMPEGMediaPlayer - Add proper AudioDataFormat negotiation AudioSink <-> libav - Parse libav's SampleFormat - Remove AudioSink interaction (moved to GLMediaPlayerImpl) - Tests (MovieSimple, MovieCube): - Add aid/vid selection - Add KeyListener for actions: seek(..), play()/pause(), setPlaySpeed(..) - Dump perf-string each 2s - TODO: - Add audio sync in AudioSink, similar to GLMediaPlayer's weighted video delay, here: drop audio frames.
Diffstat (limited to 'src/jogl/classes/jogamp/opengl')
-rw-r--r--src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java53
-rw-r--r--src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java2
-rw-r--r--src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java740
-rw-r--r--src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java27
-rw-r--r--src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java232
-rw-r--r--src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java25
6 files changed, 706 insertions, 373 deletions
diff --git a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java
index 765cda084..e14642c34 100644
--- a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java
+++ b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java
@@ -28,13 +28,13 @@
package jogamp.opengl.android.av;
import java.io.IOException;
-import java.nio.Buffer;
import javax.media.opengl.GL;
import javax.media.opengl.GLES2;
import com.jogamp.common.os.AndroidVersion;
import com.jogamp.common.os.Platform;
+import com.jogamp.opengl.util.av.GLMediaPlayer;
import com.jogamp.opengl.util.texture.Texture;
import com.jogamp.opengl.util.texture.TextureSequence;
@@ -100,7 +100,7 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
}
@Override
- protected final boolean startImpl() {
+ protected final boolean playImpl() {
if(null != mp) {
try {
mp.start();
@@ -131,22 +131,6 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
}
@Override
- protected final boolean stopImpl() {
- if(null != mp) {
- wakeUp(false);
- try {
- mp.stop();
- return true;
- } catch (IllegalStateException ise) {
- if(DEBUG) {
- ise.printStackTrace();
- }
- }
- }
- return false;
- }
-
- @Override
protected final int seekImpl(int msec) {
if(null != mp) {
mp.seekTo(msec);
@@ -165,15 +149,19 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
}
@Override
- protected final int getCurrentPositionImpl() { return null != mp ? mp.getCurrentPosition() : 0; }
-
- @Override
- protected final int getAudioPTSImpl() { return getCurrentPositionImpl(); }
+ protected final int getAudioPTSImpl() { return null != mp ? mp.getCurrentPosition() : 0; }
@Override
protected final void destroyImpl(GL gl) {
if(null != mp) {
wakeUp(false);
+ try {
+ mp.stop();
+ } catch (IllegalStateException ise) {
+ if(DEBUG) {
+ ise.printStackTrace();
+ }
+ }
mp.release();
mp = null;
}
@@ -198,8 +186,13 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
}
@Override
- protected final void initGLStreamImpl(GL gl) throws IOException {
+ protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException {
if(null!=mp && null!=urlConn) {
+ if( GLMediaPlayer.STREAM_ID_NONE == aid ) {
+ mp.setVolume(0f, 0f);
+ // FIXME: Disable audio handling
+ } // else FIXME: Select aid !
+ // Note: Both FIXMEs seem to be n/a via Android's MediaPlayer -> Switch to API level 16 MediaCodec/MediaExtractor ..
try {
final Uri uri = Uri.parse(urlConn.getURL().toExternalForm());
mp.setDataSource(StaticContext.getContext(), uri);
@@ -213,20 +206,18 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
if( null == stex ) {
throw new InternalError("XXX");
}
- final Surface surf = new Surface(stex);
- mp.setSurface(surf);
- surf.release();
mp.setSurface(null);
try {
mp.prepare();
} catch (IOException ioe) {
throw new IOException("MediaPlayer failed to process stream <"+urlConn.getURL().toExternalForm()+">: "+ioe.getMessage(), ioe);
}
+ final int r_aid = GLMediaPlayer.STREAM_ID_NONE == aid ? GLMediaPlayer.STREAM_ID_NONE : GLMediaPlayer.STREAM_ID_AUTO;
final String icodec = "android";
- updateAttributes(mp.getVideoWidth(), mp.getVideoHeight(),
- 0, 0, 0,
- 0f, 0, mp.getDuration(),
- icodec, icodec);
+ updateAttributes(GLMediaPlayer.STREAM_ID_AUTO, r_aid,
+ mp.getVideoWidth(), mp.getVideoHeight(), 0,
+ 0, 0, 0f,
+ 0, 0, mp.getDuration(), icodec, icodec);
}
}
@@ -264,8 +255,6 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl {
}
return true;
}
- @Override
- protected final void syncFrame2Audio(TextureFrame frame) {}
@Override
protected final TextureSequence.TextureFrame createTexImage(GL gl, int texName) {
diff --git a/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java
index 57d5ff625..db2146cdc 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/EGLMediaPlayerImpl.java
@@ -84,7 +84,7 @@ public abstract class EGLMediaPlayerImpl extends GLMediaPlayerImpl {
}
@Override
protected final int validateTextureCount(int desiredTextureCount) {
- return desiredTextureCount>1 ? desiredTextureCount : 2;
+ return desiredTextureCount>2 ? Math.max(4, desiredTextureCount) : 2;
}
@Override
diff --git a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java
index bc297dc21..c1cfc0d95 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java
@@ -42,6 +42,7 @@ import javax.media.opengl.GLES2;
import javax.media.opengl.GLException;
import javax.media.opengl.GLProfile;
+import com.jogamp.opengl.util.av.AudioSink;
import com.jogamp.opengl.util.av.GLMediaPlayer;
import com.jogamp.opengl.util.texture.Texture;
import com.jogamp.opengl.util.texture.TextureSequence;
@@ -62,7 +63,11 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
protected static final String unknown = "unknown";
- protected State state;
+ /** Default texture count w/o threading, value {@value}. */
+ protected static final int TEXTURE_COUNT_DEFAULT = 2;
+
+ protected volatile State state;
+ private Object stateLock = new Object();
protected int textureCount;
protected int textureTarget;
@@ -79,30 +84,72 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
protected volatile float playSpeed = 1.0f;
- /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
+ protected int vid = GLMediaPlayer.STREAM_ID_AUTO;
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
+ protected int aid = GLMediaPlayer.STREAM_ID_AUTO;
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int width = 0;
- /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int height = 0;
- /** Video fps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Video fps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected float fps = 0;
- /** Stream bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ protected int frame_period = 0;
+ /** Stream bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int bps_stream = 0;
- /** Video bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Video bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int bps_video = 0;
- /** Audio bps. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Audio bps. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int bps_audio = 0;
- /** In frames. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
- protected int totalFrames = 0;
- /** In ms. Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** In frames. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
+ protected int videoFrames = 0;
+ /** In frames. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
+ protected int audioFrames = 0;
+ /** In ms. Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected int duration = 0;
- /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected String acodec = unknown;
- /** Shall be set by the {@link #initGLStreamImpl(GL)} method implementation. */
+ /** Shall be set by the {@link #initGLStreamImpl(GL, int, int)} method implementation. */
protected String vcodec = unknown;
- protected int frameNumber = 0;
- protected int currentVideoPTS = 0;
+ protected volatile int decodedFrameCount = 0;
+ protected int presentedFrameCount = 0;
+ protected volatile int video_pts_last = 0;
+
+ /** See {@link #getAudioSink()}. Set by implementation if used from within {@link #initGLStreamImpl(GL, int, int)}! */
+ protected AudioSink audioSink = null;
+ protected boolean audioSinkPlaySpeedSet = false;
+
+ /** System Clock Reference (SCR) of first audio PTS at start time. */
+ private long audio_scr_t0 = 0;
+ private boolean audioSCR_reset = true;
+ /** System Clock Reference (SCR) of first video frame at start time. */
+ private long video_scr_t0 = 0;
+ /** System Clock Reference (SCR) PTS offset, i.e. first video PTS at start time. */
+ private int video_scr_pts = 0;
+ /** Cumulative video pts diff. */
+ private float video_dpts_cum = 0;
+ /** Cumulative video frames. */
+ private int video_dpts_count = 0;
+ /** Number of min frame count required for video cumulative sync. */
+ private static final int VIDEO_DPTS_NUM = 20;
+ /** Cumulative coefficient, value {@value}. */
+ private static final float VIDEO_DPTS_COEFF = 0.7943282f; // (float) Math.exp(Math.log(0.01) / VIDEO_DPTS_NUM);
+ /** Maximum valid video pts diff. */
+ private static final int VIDEO_DPTS_MAX = 5000; // 5s max diff
+ /** Trigger video PTS reset with given cause as bitfield. */
+ private volatile int videoSCR_reset = 0;
+
+ private final boolean isSCRCause(int bit) { return 0 != ( bit & videoSCR_reset); }
+ /** SCR reset due to: Start, Resume, Seek, .. */
+ private static final int SCR_RESET_FORCE = 1 << 0;
+ /** SCR reset due to: PlaySpeed */
+ private static final int SCR_RESET_SPEED = 1 << 1;
+
+ /** Latched video PTS reset, to wait until valid pts after invalidation of cached ones. Currently [1..{@link #VIDEO_DPTS_NUM}] frames. */
+ private int videoSCR_reset_latch = 0;
+
protected SyncedRingbuffer<TextureFrame> videoFramesFree = null;
protected SyncedRingbuffer<TextureFrame> videoFramesDecoded = null;
protected volatile TextureFrame lastFrame = null;
@@ -201,144 +248,205 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
}
@Override
- public final float getPlaySpeed() {
- return playSpeed;
- }
+ public final int getDecodedFrameCount() { return decodedFrameCount; }
+
+ @Override
+ public final int getPresentedFrameCount() { return this.presentedFrameCount; }
+
+ @Override
+ public final int getVideoPTS() { return video_pts_last; }
@Override
- public final synchronized void setPlaySpeed(float rate) {
- if(State.Uninitialized != state && setPlaySpeedImpl(rate)) {
- playSpeed = rate;
+ public final int getAudioPTS() {
+ if( State.Uninitialized != state ) {
+ return getAudioPTSImpl();
}
- if(DEBUG) { System.err.println("SetPlaySpeed: "+toString()); }
+ return 0;
}
- protected abstract boolean setPlaySpeedImpl(float rate);
-
- public final State start() {
- switch( state ) {
- case Stopped:
- /** fall-through intended */
- case Paused:
- if( startImpl() ) {
- resumeFramePusher();
- state = State.Playing;
- }
- default:
+ /** Override if not using audioSink! */
+ protected int getAudioPTSImpl() {
+ if( null != audioSink ) {
+ return audioSink.getPTS();
+ } else {
+ return 0;
}
- if(DEBUG) { System.err.println("Start: "+toString()); }
- return state;
}
- protected abstract boolean startImpl();
- public final State pause() {
- if( State.Playing == state && pauseImpl() ) {
- pauseFramePusher();
- state = State.Paused;
+ public final State getState() { return state; }
+
+ public final State play() {
+ synchronized( stateLock ) {
+ switch( state ) {
+ case Paused:
+ if( playImpl() ) {
+ resetAudioVideoSCR(SCR_RESET_FORCE);
+ resumeFramePusher();
+ if( null != audioSink ) {
+ audioSink.play();
+ }
+ state = State.Playing;
+ }
+ default:
+ }
+ if(DEBUG) { System.err.println("Start: "+toString()); }
+ return state;
}
- if(DEBUG) { System.err.println("Pause: "+toString()); }
- return state;
}
- protected abstract boolean pauseImpl();
+ protected abstract boolean playImpl();
- public final State stop() {
- switch( state ) {
- case Playing:
- /** fall-through intended */
- case Paused:
- if( stopImpl() ) {
+ public final State pause() {
+ synchronized( stateLock ) {
+ if( State.Playing == state ) {
+ State _state = state;
+ state = State.Paused;
+ if( pauseImpl() ) {
+ _state = State.Paused;
pauseFramePusher();
- state = State.Stopped;
+ if( null != audioSink ) {
+ audioSink.pause();
+ }
}
- default:
+ state = _state;
+ }
+ if(DEBUG) { System.err.println("Pause: "+toString()); }
+ return state;
}
- if(DEBUG) { System.err.println("Stop: "+toString()); }
- return state;
}
- protected abstract boolean stopImpl();
+ protected abstract boolean pauseImpl();
- @Override
- public final int getCurrentPosition() {
- if( State.Uninitialized != state ) {
- return getCurrentPositionImpl();
+ public final int seek(int msec) {
+ synchronized( stateLock ) {
+ final int pts1;
+ switch(state) {
+ case Playing:
+ case Paused:
+ final State _state = state;
+ state = State.Paused;
+ pauseFramePusher();
+ resetAudioVideoSCR(SCR_RESET_FORCE);
+ pts1 = seekImpl(msec);
+ if( null != audioSink ) {
+ audioSink.flush();
+ if( State.Playing == _state ) {
+ audioSink.play(); // cont. w/ new data
+ }
+ }
+ resumeFramePusher();
+ state = _state;
+ break;
+ default:
+ pts1 = 0;
+ }
+ if(DEBUG) { System.err.println("Seek("+msec+"): "+toString()); }
+ return pts1;
}
- return 0;
}
- protected abstract int getCurrentPositionImpl();
+ protected abstract int seekImpl(int msec);
@Override
- public final int getVideoPTS() { return currentVideoPTS; }
+ public final float getPlaySpeed() {
+ return playSpeed;
+ }
@Override
- public final int getAudioPTS() {
- if( State.Uninitialized != state ) {
- return getAudioPTSImpl();
+ public final boolean setPlaySpeed(float rate) {
+ synchronized( stateLock ) {
+ boolean res = false;
+ if(State.Uninitialized != state ) {
+ if( rate > 0.01f ) {
+ if( Math.abs(1.0f - rate) < 0.01f ) {
+ rate = 1.0f;
+ }
+ if( setPlaySpeedImpl(rate) ) {
+ resetAudioVideoSCR(SCR_RESET_SPEED);
+ playSpeed = rate;
+ if(DEBUG) { System.err.println("SetPlaySpeed: "+toString()); }
+ res = true;
+ }
+ }
+ }
+ return res;
}
- return 0;
- }
- protected abstract int getAudioPTSImpl();
-
- public final int seek(int msec) {
- final int pts1;
- switch(state) {
- case Stopped:
- case Playing:
- case Paused:
- pauseFramePusher();
- pts1 = seekImpl(msec);
- currentVideoPTS=pts1;
- resumeFramePusher();
- break;
- default:
- pts1 = 0;
+ }
+ /**
+ * Override if not using AudioSink, or AudioSink's {@link AudioSink#setPlaySpeed(float)} is not sufficient!
+ * <p>
+ * AudioSink shall respect <code>!audioSinkPlaySpeedSet</code> to determine data_size
+ * at {@link AudioSink#enqueueData(com.jogamp.opengl.util.av.AudioSink.AudioFrame)}.
+ * </p>
+ */
+ protected boolean setPlaySpeedImpl(float rate) {
+ if( null != audioSink ) {
+ audioSinkPlaySpeedSet = audioSink.setPlaySpeed(rate);
}
- if(DEBUG) { System.err.println("Seek("+msec+"): "+toString()); }
- return pts1;
+ // still true, even if audioSink rejects command since we deal w/ video sync
+ // and AudioSink w/ audioSinkPlaySpeedSet at enqueueData(..).
+ return true;
}
- protected abstract int seekImpl(int msec);
-
- public final State getState() { return state; }
-
+
@Override
- public final State initGLStream(GL gl, int reqTextureCount, URLConnection urlConn) throws IllegalStateException, GLException, IOException {
- if(State.Uninitialized != state) {
- throw new IllegalStateException("Instance not in state "+State.Uninitialized+", but "+state+", "+this);
- }
- this.urlConn = urlConn;
- if (this.urlConn != null) {
- try {
- if( null != gl ) {
- removeAllTextureFrames(gl);
- textureCount = validateTextureCount(reqTextureCount);
- if( textureCount < 2 ) {
- throw new InternalError("Validated texture count < 2: "+textureCount);
- }
- initGLStreamImpl(gl); // also initializes width, height, .. etc
- videoFramesFree = new SyncedRingbuffer<TextureFrame>(createTexFrames(gl, textureCount), true /* full */);
- if( 2 < textureCount ) {
- videoFramesDecoded = new SyncedRingbuffer<TextureFrame>(new TextureFrame[textureCount], false /* full */);
- framePusher = new FramePusher(gl, requiresOffthreadGLCtx());
- framePusher.doStart();
- } else {
- videoFramesDecoded = null;
+ public final State initGLStream(GL gl, int reqTextureCount, URLConnection urlConn, int vid, int aid) throws IllegalStateException, GLException, IOException {
+ synchronized( stateLock ) {
+ if(State.Uninitialized != state) {
+ throw new IllegalStateException("Instance not in state "+State.Uninitialized+", but "+state+", "+this);
+ }
+ decodedFrameCount = 0;
+ presentedFrameCount = 0;
+ this.urlConn = urlConn;
+ if (this.urlConn != null) {
+ try {
+ if( null != gl ) {
+ removeAllTextureFrames(gl);
+ textureCount = validateTextureCount(reqTextureCount);
+ if( textureCount < TEXTURE_COUNT_DEFAULT ) {
+ throw new InternalError("Validated texture count < "+TEXTURE_COUNT_DEFAULT+": "+textureCount);
+ }
+ initGLStreamImpl(gl, vid, aid); // also initializes width, height, .. etc
+ videoFramesFree = new SyncedRingbuffer<TextureFrame>(createTexFrames(gl, textureCount), true /* full */);
+ if( TEXTURE_COUNT_DEFAULT < textureCount ) {
+ videoFramesDecoded = new SyncedRingbuffer<TextureFrame>(new TextureFrame[textureCount], false /* full */);
+ framePusher = new FramePusher(gl, requiresOffthreadGLCtx());
+ framePusher.doStart();
+ } else {
+ videoFramesDecoded = null;
+ }
+ lastFrame = videoFramesFree.getBlocking(false /* clearRef */ );
+ state = State.Paused;
}
- lastFrame = videoFramesFree.getBlocking(false /* clearRef */ );
+ return state;
+ } catch (Throwable t) {
+ throw new GLException("Error initializing GL resources", t);
}
- state = State.Stopped;
- return state;
- } catch (Throwable t) {
- throw new GLException("Error initializing GL resources", t);
}
+ return state;
}
- return state;
}
+ /**
+ * Implementation shall set the following set of data here
+ * @see #vid
+ * @see #aid
+ * @see #width
+ * @see #height
+ * @see #fps
+ * @see #bps_stream
+ * @see #videoFrames
+ * @see #audioFrames
+ * @see #acodec
+ * @see #vcodec
+ */
+ protected abstract void initGLStreamImpl(GL gl, int vid, int aid) throws IOException;
+
/**
* Returns the validated number of textures to be handled.
* <p>
- * Default is always 2 textures, last texture and the decoding texture.
+ * Default is 2 textures w/o threading, last texture and the decoding texture.
+ * </p>
+ * <p>
+ * &gt; 2 textures is used for threaded decoding, a minimum of 4 textures seems reasonable in this case.
* </p>
*/
protected int validateTextureCount(int desiredTextureCount) {
- return 2;
+ return TEXTURE_COUNT_DEFAULT;
}
protected boolean requiresOffthreadGLCtx() { return false; }
@@ -405,6 +513,18 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
mustFlipVertically);
}
+ protected void destroyTexFrame(GL gl, TextureFrame frame) {
+ frame.getTexture().destroy(gl);
+ }
+
+ @Override
+ public final TextureFrame getLastTexture() throws IllegalStateException {
+ if(State.Uninitialized == state) {
+ throw new IllegalStateException("Instance not initialized: "+this);
+ }
+ return lastFrame;
+ }
+
private final void removeAllTextureFrames(GL gl) {
if( null != videoFramesFree ) {
final TextureFrame[] texFrames = videoFramesFree.getArray();
@@ -417,79 +537,209 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
destroyTexFrame(gl, frame);
texFrames[i] = null;
}
+ System.err.println(Thread.currentThread().getName()+"> Clear TexFrame["+i+"]: "+frame+" -> null");
}
}
textureCount=0;
}
- protected void destroyTexFrame(GL gl, TextureFrame frame) {
- frame.getTexture().destroy(gl);
- }
-
- /**
- * Implementation shall set the following set of data here
- * @param gl TODO
- * @see #width
- * @see #height
- * @see #fps
- * @see #bps_stream
- * @see #totalFrames
- * @see #acodec
- * @see #vcodec
- */
- protected abstract void initGLStreamImpl(GL gl) throws IOException;
@Override
- public final TextureFrame getLastTexture() throws IllegalStateException {
- if(State.Uninitialized == state) {
- throw new IllegalStateException("Instance not initialized: "+this);
+ public final TextureFrame getNextTexture(GL gl, boolean blocking) throws IllegalStateException {
+ synchronized( stateLock ) {
+ if(State.Uninitialized == state) {
+ throw new IllegalStateException("Instance not initialized: "+this);
+ }
+ if(State.Playing == state) {
+ TextureFrame nextFrame = null;
+ boolean ok = true;
+ boolean dropFrame = false;
+ try {
+ do {
+ if( TEXTURE_COUNT_DEFAULT < textureCount ) {
+ nextFrame = videoFramesDecoded.getBlocking(false /* clearRef */ );
+ } else {
+ nextFrame = videoFramesFree.getBlocking(false /* clearRef */ );
+ if( getNextTextureImpl(gl, nextFrame, blocking) ) {
+ newFrameAvailable(nextFrame);
+ } else {
+ ok = false;
+ }
+ }
+ if( ok ) {
+ presentedFrameCount++;
+ final int video_pts;
+ if( 0 != videoSCR_reset ) {
+ if( isSCRCause(SCR_RESET_FORCE) ) {
+ videoSCR_reset_latch = VIDEO_DPTS_NUM / 2;
+ resetVideoDPTS();
+ resetAllVideoPTS();
+ } else {
+ // SCR_RESET_SPEED
+ videoSCR_reset_latch = 1;
+ }
+ videoSCR_reset = 0;
+ video_pts = TextureFrame.INVALID_PTS;
+ } else {
+ video_pts = nextFrame.getPTS();
+ }
+ if( video_pts != TextureFrame.INVALID_PTS ) {
+ final int frame_period_last = video_pts - video_pts_last; // rendering loop interrupted ?
+ if( videoSCR_reset_latch > 0 || frame_period_last > frame_period*10 ) {
+ if( videoSCR_reset_latch > 0 ) {
+ videoSCR_reset_latch--;
+ }
+ setFirstVideoPTS2SCR( video_pts );
+ }
+ final int scr_pts = video_scr_pts +
+ (int) ( ( System.currentTimeMillis() - video_scr_t0 ) * playSpeed );
+ final int d_vpts = video_pts - scr_pts;
+ if( -VIDEO_DPTS_MAX > d_vpts || d_vpts > VIDEO_DPTS_MAX ) {
+ if( DEBUG ) {
+ System.err.println( getPerfStringImpl( scr_pts, video_pts, d_vpts, 0 ) );
+ }
+ } else {
+ video_dpts_count++;
+ video_dpts_cum = d_vpts + VIDEO_DPTS_COEFF * video_dpts_cum;
+ final int video_dpts_avg_diff = getVideoDPTSAvg();
+ if( DEBUG ) {
+ System.err.println( getPerfStringImpl( scr_pts, video_pts, d_vpts, video_dpts_avg_diff ) );
+ }
+ if( blocking && syncAVRequired() ) {
+ if( !syncAV( (int) ( video_dpts_avg_diff / playSpeed + 0.5f ) ) ) {
+ resetVideoDPTS();
+ dropFrame = true;
+ }
+ }
+ video_pts_last = video_pts;
+ }
+ }
+ final TextureFrame _lastFrame = lastFrame;
+ lastFrame = nextFrame;
+ videoFramesFree.putBlocking(_lastFrame);
+ }
+ } while( dropFrame );
+ } catch (InterruptedException e) {
+ ok = false;
+ e.printStackTrace();
+ } finally {
+ if( !ok && null != nextFrame ) { // put back
+ videoFramesFree.put(nextFrame);
+ }
+ }
+ }
+ return lastFrame;
}
- return lastFrame;
}
+ protected abstract boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking);
+ protected boolean syncAVRequired() { return false; }
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Note: All {@link AudioSink} operations are performed from {@link GLMediaPlayerImpl},
+ * i.e. {@link #play()}, {@link #pause()}, {@link #seek(int)}, {@link #setPlaySpeed(float)}, {@link #getAudioPTS()}.
+ * </p>
+ * <p>
+ * Implementations using an {@link AudioSink} shall write it's instance to {@link #audioSink}
+ * from within their {@link #initGLStreamImpl(GL, int, int)} implementation.
+ * </p>
+ */
@Override
- public final synchronized TextureFrame getNextTexture(GL gl, boolean blocking) throws IllegalStateException {
- if(State.Uninitialized == state) {
- throw new IllegalStateException("Instance not initialized: "+this);
+ public final AudioSink getAudioSink() { return audioSink; }
+
+ /**
+ * To be called from implementation at 1st PTS after start
+ * w/ current pts value in milliseconds.
+ * @param audio_scr_t0
+ */
+ protected void setFirstAudioPTS2SCR(int pts) {
+ if( audioSCR_reset ) {
+ audio_scr_t0 = System.currentTimeMillis() - pts;
+ audioSCR_reset = false;
}
- if(State.Playing == state) {
- TextureFrame nextFrame = null;
- boolean ok = true;
- try {
- if( 2 < textureCount ) {
- nextFrame = videoFramesDecoded.getBlocking(false /* clearRef */ );
- } else {
- nextFrame = videoFramesFree.getBlocking(false /* clearRef */ );
- if( getNextTextureImpl(gl, nextFrame, blocking) ) {
- newFrameAvailable(nextFrame);
- } else {
- ok = false;
- }
- }
- if( ok ) {
- currentVideoPTS = nextFrame.getPTS();
- if( blocking ) {
- syncFrame2Audio(nextFrame);
- }
- final TextureFrame _lastFrame = lastFrame;
- lastFrame = nextFrame;
- videoFramesFree.putBlocking(_lastFrame);
- }
- } catch (InterruptedException e) {
- ok = false;
- e.printStackTrace();
- } finally {
- if( !ok && null != nextFrame ) { // put back
- videoFramesFree.put(nextFrame);
- }
+ }
+ private void setFirstVideoPTS2SCR(int pts) {
+ // video_scr_t0 = System.currentTimeMillis() - pts;
+ video_scr_t0 = System.currentTimeMillis();
+ video_scr_pts = pts;
+ }
+ private void resetAllVideoPTS() {
+ if( null != videoFramesFree ) {
+ final TextureFrame[] texFrames = videoFramesFree.getArray();
+ for(int i=0; i<texFrames.length; i++) {
+ final TextureFrame frame = texFrames[i];
+ frame.setPTS(TextureFrame.INVALID_PTS);
+ }
+ }
+ }
+ private void resetVideoDPTS() {
+ video_dpts_cum = 0;
+ video_dpts_count = 0;
+ }
+ private final int getVideoDPTSAvg() {
+ if( video_dpts_count < VIDEO_DPTS_NUM ) {
+ return 0;
+ } else {
+ return (int) ( video_dpts_cum * (1.0f - VIDEO_DPTS_COEFF) + 0.5f );
+ }
+ }
+
+ private void resetAudioVideoSCR(int cause) {
+ audioSCR_reset = true;
+ videoSCR_reset |= cause;
+ }
+
+ /**
+ * Synchronizes A-V.
+ * <p>
+ * https://en.wikipedia.org/wiki/Audio_to_video_synchronization
+ * <pre>
+ * d_av = v_pts - a_pts;
+ * </pre>
+ * </p>
+ * <p>
+ * Recommendation of audio/video pts time lead/lag at production:
+ * <ul>
+ * <li>Overall: +40ms and -60ms audio ahead video / audio after video</li>
+ * <li>Each stage: +5ms and -15ms. audio ahead video / audio after video</li>
+ * </ul>
+ * </p>
+ * <p>
+ * Recommendation of av pts time lead/lag at presentation:
+ * <ul>
+ * <li>TV: +15ms and -45ms. audio ahead video / audio after video.</li>
+ * <li>Film: +22ms and -22ms. audio ahead video / audio after video.</li>
+ * </ul>
+ * </p>
+ * <p>
+ * Maybe implemented as follows:
+ * <pre>
+ * d_av = vpts - apts;
+ * d_av < -22: audio after video == video ahead audio -> drop
+ * d_av > 22: audio ahead video == video after audio -> sleep(d_av - 10)
+ * </pre>
+ * </p>
+ * <p>
+ * Returns true if audio is ahead of video, otherwise false (video is ahead of audio).
+ * In case of the latter (false), the video frame shall be dropped!
+ * </p>
+ * @param frame
+ * @return true if audio is ahead of video, otherwise false (video is ahead of audio)
+ */
+ protected boolean syncAV(int d_vpts) {
+ if( d_vpts > 22 ) {
+ if( DEBUG ) {
+ System.err.println("V (sleep): "+(d_vpts - 22 / 2)+" ms");
}
+ try {
+ Thread.sleep( d_vpts - 22 / 2 );
+ } catch (InterruptedException e) { }
}
- return lastFrame;
+ return true;
}
- protected abstract boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking);
- protected abstract void syncFrame2Audio(TextureFrame frame);
private final void newFrameAvailable(TextureFrame frame) {
- frameNumber++;
+ decodedFrameCount++;
synchronized(eventListenersLock) {
for(Iterator<GLMediaEventListener> i = eventListeners.iterator(); i.hasNext(); ) {
i.next().newFrameAvailable(this, frame, System.currentTimeMillis());
@@ -500,6 +750,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
class FramePusher extends Thread {
private volatile boolean isRunning = false;
private volatile boolean isActive = false;
+ private volatile boolean isBlocked = false;
private volatile boolean shallPause = true;
private volatile boolean shallStop = false;
@@ -560,6 +811,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
public synchronized void doPause() {
if( isActive ) {
shallPause = true;
+ if( isBlocked && isActive ) {
+ this.interrupt();
+ }
while( isActive ) {
try {
this.wait(); // wait until paused
@@ -595,6 +849,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
public synchronized void doStop() {
if( isRunning ) {
shallStop = true;
+ if( isBlocked && isRunning ) {
+ this.interrupt();
+ }
while( isRunning ) {
this.notify(); // wake-up pause-block (opt)
try {
@@ -629,7 +886,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
try {
this.wait(); // wait until resumed
} catch (InterruptedException e) {
- e.printStackTrace();
+ if( !shallPause ) {
+ e.printStackTrace();
+ }
}
}
isActive = true;
@@ -639,23 +898,30 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
if( !shallStop ) {
TextureFrame nextFrame = null;
- boolean ok = false;
try {
- nextFrame = videoFramesFree.getBlocking(true /* clearRef */ );
+ isBlocked = true;
+ nextFrame = videoFramesFree.getBlocking(false /* clearRef */ );
+ isBlocked = false;
+ nextFrame.setPTS( TextureFrame.INVALID_PTS ); // mark invalid until processed!
if( getNextTextureImpl(gl, nextFrame, true) ) {
- gl.glFinish();
- videoFramesDecoded.putBlocking(nextFrame);
- newFrameAvailable(nextFrame);
- ok = true;
+ // gl.glFinish();
+ gl.glFlush(); // even better: sync object!
+ if( !videoFramesDecoded.put(nextFrame) ) {
+ throw new InternalError("XXX: "+GLMediaPlayerImpl.this);
+ }
+ final TextureFrame _nextFrame = nextFrame;
+ nextFrame = null;
+ newFrameAvailable(_nextFrame);
}
} catch (InterruptedException e) {
+ isBlocked = false;
if( !shallStop && !shallPause ) {
e.printStackTrace(); // oops
shallPause = false;
shallStop = true;
}
} finally {
- if( !ok && null != nextFrame ) { // put back
+ if( null != nextFrame ) { // put back
videoFramesFree.put(nextFrame);
}
}
@@ -689,10 +955,18 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
}
}
- protected final void updateAttributes(int width, int height, int bps_stream, int bps_video, int bps_audio,
- float fps, int totalFrames, int duration,
- String vcodec, String acodec) {
+ protected final void updateAttributes(int vid, int aid, int width, int height, int bps_stream,
+ int bps_video, int bps_audio, float fps,
+ int videoFrames, int audioFrames, int duration, String vcodec, String acodec) {
int event_mask = 0;
+ if( this.vid != vid ) {
+ event_mask |= GLMediaEventListener.EVENT_CHANGE_VID;
+ this.vid = vid;
+ }
+ if( this.aid != aid ) {
+ event_mask |= GLMediaEventListener.EVENT_CHANGE_AID;
+ this.aid = aid;
+ }
if( this.width != width || this.height != height ) {
event_mask |= GLMediaEventListener.EVENT_CHANGE_SIZE;
this.width = width;
@@ -701,6 +975,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
if( this.fps != fps ) {
event_mask |= GLMediaEventListener.EVENT_CHANGE_FPS;
this.fps = fps;
+ this.frame_period = (int) ( 1000f / fps + 0.5f );
}
if( this.bps_stream != bps_stream || this.bps_video != bps_video || this.bps_audio != bps_audio ) {
event_mask |= GLMediaEventListener.EVENT_CHANGE_BPS;
@@ -708,9 +983,10 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
this.bps_video = bps_video;
this.bps_audio = bps_audio;
}
- if( this.totalFrames != totalFrames || this.duration != duration ) {
+ if( this.videoFrames != videoFrames || this.audioFrames != audioFrames || this.duration != duration ) {
event_mask |= GLMediaEventListener.EVENT_CHANGE_LENGTH;
- this.totalFrames = totalFrames;
+ this.videoFrames = videoFrames;
+ this.audioFrames = audioFrames;
this.duration = duration;
}
if( (null!=acodec && acodec.length()>0 && !this.acodec.equals(acodec)) ) {
@@ -736,78 +1012,120 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
}
@Override
- public final synchronized State destroy(GL gl) {
- destroyFramePusher();
- destroyImpl(gl);
- removeAllTextureFrames(gl);
- state = State.Uninitialized;
- return state;
+ public final State destroy(GL gl) {
+ synchronized( stateLock ) {
+ destroyFramePusher();
+ destroyImpl(gl);
+ removeAllTextureFrames(gl);
+ state = State.Uninitialized;
+ return state;
+ }
}
protected abstract void destroyImpl(GL gl);
@Override
- public final synchronized URLConnection getURLConnection() {
+ public final URLConnection getURLConnection() {
return urlConn;
}
@Override
- public final synchronized String getVideoCodec() {
+ public final int getVID() { return vid; }
+
+ @Override
+ public final int getAID() { return aid; }
+
+ @Override
+ public final String getVideoCodec() {
return vcodec;
}
@Override
- public final synchronized String getAudioCodec() {
+ public final String getAudioCodec() {
return acodec;
}
@Override
- public final synchronized long getTotalFrames() {
- return totalFrames;
+ public final int getVideoFrames() {
+ return videoFrames;
+ }
+
+ public final int getAudioFrames() {
+ return audioFrames;
}
@Override
- public final synchronized int getDuration() {
+ public final int getDuration() {
return duration;
}
@Override
- public final synchronized long getStreamBitrate() {
+ public final long getStreamBitrate() {
return bps_stream;
}
@Override
- public final synchronized int getVideoBitrate() {
+ public final int getVideoBitrate() {
return bps_video;
}
@Override
- public final synchronized int getAudioBitrate() {
+ public final int getAudioBitrate() {
return bps_audio;
}
@Override
- public final synchronized float getFramerate() {
+ public final float getFramerate() {
return fps;
}
@Override
- public final synchronized int getWidth() {
+ public final int getWidth() {
return width;
}
@Override
- public final synchronized int getHeight() {
+ public final int getHeight() {
return height;
}
@Override
- public final synchronized String toString() {
- final float ct = getCurrentPosition() / 1000.0f, tt = getDuration() / 1000.0f;
+ public final String toString() {
+ final float tt = getDuration() / 1000.0f;
final String loc = ( null != urlConn ) ? urlConn.getURL().toExternalForm() : "<undefined stream>" ;
- return "GLMediaPlayer["+state+", "+frameNumber+"/"+totalFrames+" frames, "+ct+"/"+tt+"s, speed "+playSpeed+", "+bps_stream+" bps, "+
- "Texture[count "+textureCount+", target "+toHexString(textureTarget)+", format "+toHexString(textureFormat)+", type "+toHexString(textureType)+"], "+
- "Stream[Video[<"+vcodec+">, "+width+"x"+height+", "+fps+" fps, "+bps_video+" bsp], "+
- "Audio[<"+acodec+">, "+bps_audio+" bsp]], "+loc+"]";
+ final int freeVideoFrames = null != videoFramesFree ? videoFramesFree.size() : 0;
+ final int decVideoFrames = null != videoFramesDecoded ? videoFramesDecoded.size() : 0;
+ return "GLMediaPlayer["+state+", frames[p "+presentedFrameCount+", d "+decodedFrameCount+", t "+videoFrames+" ("+tt+" s)], "+
+ "speed "+playSpeed+", "+bps_stream+" bps, "+
+ "Texture[count "+textureCount+", free "+freeVideoFrames+", dec "+decVideoFrames+", target "+toHexString(textureTarget)+", format "+toHexString(textureFormat)+", type "+toHexString(textureType)+"], "+
+ "Video[id "+vid+", <"+vcodec+">, "+width+"x"+height+", "+fps+" fps, "+bps_video+" bps], "+
+ "Audio[id "+aid+", <"+acodec+">, "+bps_audio+" bps, "+audioFrames+" frames], uri "+loc+"]";
+ }
+
+ @Override
+ public final String getPerfString() {
+ final int scr_pts = video_scr_pts +
+ (int) ( ( System.currentTimeMillis() - video_scr_t0 ) * playSpeed );
+ final int d_vpts = video_pts_last - scr_pts;
+ return getPerfStringImpl( scr_pts, video_pts_last, d_vpts, getVideoDPTSAvg() );
+ }
+ private final String getPerfStringImpl(final int scr_pts, final int video_pts, final int d_vpts, final int video_dpts_avg_diff) {
+ final float tt = getDuration() / 1000.0f;
+ final int audio_scr = (int) ( ( System.currentTimeMillis() - audio_scr_t0 ) * playSpeed );
+ final int audio_pts = getAudioPTSImpl();
+ final int d_apts = audio_pts - audio_scr;
+ final String audioSinkInfo;
+ final AudioSink audioSink = getAudioSink();
+ if( null != audioSink ) {
+ audioSinkInfo = "AudioSink[frames [d "+audioSink.getEnqueuedFrameCount()+", q "+audioSink.getQueuedFrameCount()+", f "+audioSink.getFreeFrameCount()+"], time "+audioSink.getQueuedTime()+", bytes "+audioSink.getQueuedByteCount()+"]";
+ } else {
+ audioSinkInfo = "";
+ }
+ final int freeVideoFrames = null != videoFramesFree ? videoFramesFree.size() : 0;
+ final int decVideoFrames = null != videoFramesDecoded ? videoFramesDecoded.size() : 0;
+ return state+", frames[p "+presentedFrameCount+", d "+decodedFrameCount+", t "+videoFrames+" ("+tt+" s)], "+
+ "speed " + playSpeed+", vSCR "+scr_pts+", vpts "+video_pts+", dSCR["+d_vpts+", avrg "+video_dpts_avg_diff+"], "+
+ "aSCR "+audio_scr+", apts "+audio_pts+" ( "+d_apts+" ), "+audioSinkInfo+
+ ", Texture[count "+textureCount+", free "+freeVideoFrames+", dec "+decVideoFrames+"]";
}
@Override
@@ -831,7 +1149,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer {
}
@Override
- public final synchronized GLMediaEventListener[] getEventListeners() {
+ public final GLMediaEventListener[] getEventListeners() {
synchronized(eventListenersLock) {
return eventListeners.toArray(new GLMediaEventListener[eventListeners.size()]);
}
diff --git a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
index f1ce42257..5d70ca33d 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java
@@ -38,6 +38,7 @@ import jogamp.opengl.util.av.GLMediaPlayerImpl;
import com.jogamp.common.nio.Buffers;
import com.jogamp.common.util.IOUtil;
+import com.jogamp.opengl.util.av.GLMediaPlayer;
import com.jogamp.opengl.util.texture.Texture;
import com.jogamp.opengl.util.texture.TextureData;
import com.jogamp.opengl.util.texture.TextureIO;
@@ -62,7 +63,7 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
}
@Override
- protected final boolean startImpl() {
+ protected final boolean playImpl() {
pos_start = (int)System.currentTimeMillis();
return true;
}
@@ -73,11 +74,6 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
}
@Override
- protected final boolean stopImpl() {
- return true;
- }
-
- @Override
protected final int seekImpl(int msec) {
pos_ms = msec;
validatePos();
@@ -86,20 +82,16 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
@Override
protected final boolean getNextTextureImpl(GL gl, TextureFrame nextFrame, boolean blocking) {
+ nextFrame.setPTS( getAudioPTSImpl() );
return true;
}
- @Override
- protected final void syncFrame2Audio(TextureFrame frame) { }
@Override
- protected final int getCurrentPositionImpl() {
+ protected final int getAudioPTSImpl() {
pos_ms = (int)System.currentTimeMillis() - pos_start;
validatePos();
return pos_ms;
}
- @Override
- protected final int getAudioPTSImpl() { return getCurrentPositionImpl(); }
-
@Override
protected final void destroyImpl(GL gl) {
@@ -110,7 +102,7 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
}
@Override
- protected final void initGLStreamImpl(GL gl) throws IOException {
+ protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException {
try {
URLConnection urlConn = IOUtil.getResource("jogl/util/data/av/test-ntsc01-160x90.png", this.getClass().getClassLoader());
if(null != urlConn) {
@@ -136,13 +128,14 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl {
GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, false,
false, false, buffer, null);
}
+ final int r_aid = GLMediaPlayer.STREAM_ID_NONE == aid ? GLMediaPlayer.STREAM_ID_NONE : GLMediaPlayer.STREAM_ID_AUTO;
final float _fps = 24f;
final int _duration = 10*60*1000; // msec
final int _totalFrames = (int) ( (_duration/1000)*_fps );
- updateAttributes(_w, _h,
- 0, 0, 0,
- _fps, _totalFrames, _duration,
- "png-static", null);
+ updateAttributes(GLMediaPlayer.STREAM_ID_AUTO, r_aid,
+ _w, _h, 0,
+ 0, 0, _fps,
+ _totalFrames, 0, _duration, "png-static", null);
}
@Override
diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
index 83a5960f1..dc7ceae39 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java
@@ -42,9 +42,11 @@ import com.jogamp.common.util.VersionNumber;
import com.jogamp.gluegen.runtime.ProcAddressTable;
import com.jogamp.opengl.util.GLPixelStorageModes;
import com.jogamp.opengl.util.av.AudioSink;
+import com.jogamp.opengl.util.av.AudioSink.AudioDataFormat;
+import com.jogamp.opengl.util.av.AudioSink.AudioDataType;
import com.jogamp.opengl.util.av.AudioSinkFactory;
+import com.jogamp.opengl.util.av.GLMediaPlayer;
import com.jogamp.opengl.util.texture.Texture;
-import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame;
import jogamp.opengl.GLContextImpl;
import jogamp.opengl.util.av.GLMediaPlayerImpl;
@@ -136,13 +138,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
( vers >> 8 ) & 0xFF,
( vers >> 0 ) & 0xFF );
}
+
+ //
+ // General
+ //
+
+ protected long moviePtr = 0;
//
// Video
//
- protected long moviePtr = 0;
- protected GLPixelStorageModes psm;
protected PixelFormat vPixelFmt = null;
protected int vPlanes = 0;
protected int vBitsPerPixel = 0;
@@ -152,15 +158,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
protected int texWidth, texHeight; // overall (stuffing planes in one texture)
protected ByteBuffer texCopy;
protected String singleTexComp = "r";
+ protected GLPixelStorageModes psm;
//
// Audio
//
- protected final int AudioFrameCount = 8;
- protected final AudioSink audioSink;
- protected final int maxAvailableAudio;
- protected AudioSink.AudioDataFormat chosenAudioFormat;
+ protected static final int AFRAMES_PER_VFRAME = 8;
+ protected int aFrameCount = 0;
+ protected SampleFormat aSampleFmt = null;
+ protected AudioSink.AudioDataFormat avChosenAudioFormat;
+ protected AudioSink.AudioDataFormat sinkChosenAudioFormat;
public FFMPEGMediaPlayer() {
if(!available) {
@@ -171,12 +179,11 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
throw new GLException("Couldn't create FFMPEGInstance");
}
psm = new GLPixelStorageModes();
- audioSink = AudioSinkFactory.createDefault();
- maxAvailableAudio = audioSink.getQueuedByteCount();
+ audioSink = null;
}
@Override
protected final int validateTextureCount(int desiredTextureCount) {
- return desiredTextureCount>1 ? desiredTextureCount : 2;
+ return desiredTextureCount>2 ? Math.max(4, desiredTextureCount) : 2;
}
@Override
protected final boolean requiresOffthreadGLCtx() { return true; }
@@ -187,10 +194,18 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
destroyInstance0(moviePtr);
moviePtr = 0;
}
+ destroyAudioSink();
+ }
+ private final void destroyAudioSink() {
+ final AudioSink _audioSink = audioSink;
+ if( null != _audioSink ) {
+ audioSink = null;
+ _audioSink.destroy();
+ }
}
@Override
- protected final void initGLStreamImpl(GL gl) throws IOException {
+ protected final void initGLStreamImpl(GL gl, int vid, int aid) throws IOException {
if(0==moviePtr) {
throw new GLException("FFMPEG native instance null");
}
@@ -209,11 +224,32 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
}
final String urlS=urlConn.getURL().toExternalForm();
+
+ aFrameCount = AFRAMES_PER_VFRAME * textureCount + AFRAMES_PER_VFRAME/2;
- chosenAudioFormat = audioSink.initSink(audioSink.getPreferredFormat(), AudioFrameCount);
System.err.println("setURL: p1 "+this);
- setStream0(moviePtr, urlS, -1, -1, AudioFrameCount); // issues updateAttributes*(..)
- System.err.println("setURL: p2 "+this);
+ destroyAudioSink();
+ AudioSink _audioSink;
+ if( GLMediaPlayer.STREAM_ID_NONE == aid ) {
+ _audioSink = AudioSinkFactory.createNull();
+ } else {
+ _audioSink = AudioSinkFactory.createDefault();
+ }
+ final AudioDataFormat preferredAudioFormat = _audioSink.getPreferredFormat();
+ // setStream(..) issues updateAttributes*(..), and defines avChosenAudioFormat, vid, aid, .. etc
+ setStream0(moviePtr, urlS, vid, aid, aFrameCount, preferredAudioFormat.channelCount, preferredAudioFormat.sampleRate);
+ // final int audioBytesPerFrame = bps_audio/8000 * frame_period * textureCount;
+
+ System.err.println("setURL: p2 preferred "+preferredAudioFormat+", avChosen "+avChosenAudioFormat+", "+this);
+ sinkChosenAudioFormat = _audioSink.initSink(avChosenAudioFormat, aFrameCount);
+ System.err.println("setURL: p3 avChosen "+avChosenAudioFormat+", chosen "+sinkChosenAudioFormat);
+ if( null == sinkChosenAudioFormat ) {
+ System.err.println("AudioSink "+_audioSink.getClass().getName()+" does not support "+avChosenAudioFormat+", using Null");
+ _audioSink.destroy();
+ _audioSink = AudioSinkFactory.createNull();
+ sinkChosenAudioFormat = _audioSink.initSink(avChosenAudioFormat, aFrameCount);
+ }
+ audioSink = _audioSink;
int tf, tif=GL.GL_RGBA; // texture format and internal format
switch(vBytesPerPixelPerPlane) {
@@ -256,7 +292,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
private void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane,
int lSz0, int lSz1, int lSz2,
- int tWd0, int tWd1, int tWd2) {
+ int tWd0, int tWd1, int tWd2,
+ int sampleFmt, int sampleRate, int channels) {
vPixelFmt = PixelFormat.valueOf(pixFmt);
vPlanes = planes;
vBitsPerPixel = bitsPerPixel;
@@ -286,12 +323,53 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
default: // FIXME: Add more planar formats !
throw new RuntimeException("Unsupported pixelformat: "+vPixelFmt);
}
+
+ aSampleFmt = SampleFormat.valueOf(sampleFmt);
+ final int sampleSize;
+ final boolean signed, fixedP;
+ switch( aSampleFmt ) {
+ case S32:
+ case S32P:
+ sampleSize = 32;
+ signed = true;
+ fixedP = true;
+ break;
+ case S16:
+ case S16P:
+ sampleSize = 16;
+ signed = true;
+ fixedP = true;
+ break;
+ case U8:
+ case U8P:
+ sampleSize = 8;
+ signed = false;
+ fixedP = true;
+ break;
+ case DBL:
+ case DBLP:
+ sampleSize = 64;
+ signed = true;
+ fixedP = true;
+ break;
+ case FLT:
+ case FLTP:
+ sampleSize = 32;
+ signed = true;
+ fixedP = true;
+ break;
+ default: // FIXME: Add more planar formats !
+ throw new RuntimeException("Unsupported sampleformat: "+aSampleFmt);
+ }
+ avChosenAudioFormat = new AudioDataFormat(AudioDataType.PCM, sampleRate, sampleSize, channels, signed, fixedP, true /* littleEndian */);
+
if(DEBUG) {
- System.err.println("XXX0: fmt "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane);
+ System.err.println("audio: fmt "+aSampleFmt+", "+avChosenAudioFormat);
+ System.err.println("video: fmt "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane);
for(int i=0; i<3; i++) {
- System.err.println("XXX0 "+i+": "+vTexWidth[i]+"/"+vLinesize[i]);
+ System.err.println("video: "+i+": "+vTexWidth[i]+"/"+vLinesize[i]);
}
- System.err.println("XXX0 total tex "+texWidth+"x"+texHeight);
+ System.err.println("video: total tex "+texWidth+"x"+texHeight);
}
}
@@ -355,54 +433,27 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
}
@Override
- protected final synchronized int getCurrentPositionImpl() {
- return 0!=moviePtr ? getVideoPTS0(moviePtr) : 0;
- }
-
- @Override
- public final int getAudioPTSImpl() { return 0; }
-
- @Override
- protected final synchronized boolean setPlaySpeedImpl(float rate) {
- return true;
- }
-
- @Override
- public final synchronized boolean startImpl() {
+ public final boolean playImpl() {
if(0==moviePtr) {
return false;
}
return true;
}
- /** @return time position after issuing the command */
@Override
- public final synchronized boolean pauseImpl() {
+ public final boolean pauseImpl() {
if(0==moviePtr) {
return false;
}
return true;
}
- /** @return time position after issuing the command */
- @Override
- public final synchronized boolean stopImpl() {
- if(0==moviePtr) {
- return false;
- }
- return true;
- }
-
- /** @return time position after issuing the command */
@Override
protected final synchronized int seekImpl(int msec) {
if(0==moviePtr) {
throw new GLException("FFMPEG native instance null");
}
- int pts0 = getVideoPTS0(moviePtr);
- int pts1 = seek0(moviePtr, msec);
- System.err.println("Seek: "+pts0+" -> "+msec+" : "+pts1);
- return pts1;
+ return seek0(moviePtr, msec);
}
@Override
@@ -427,7 +478,6 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
psm.restore(gl);
}
if( 0 < avPTS ) {
- vSTS = avPTS;
nextFrame.setPTS(avPTS);
return true;
} else {
@@ -436,50 +486,15 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
}
private final void pushSound(ByteBuffer sampleData, int data_size, int audio_pts) {
- aSTS = audio_pts;
- final AudioSink.AudioFrame frame = new AudioSink.AudioFrame(sampleData, data_size, audio_pts);
- if( audioSink.isDataAvailable(frame.dataSize) ) {
- audioSink.writeData(frame);
+ setFirstAudioPTS2SCR( audio_pts );
+ if( 1.0f == playSpeed || audioSinkPlaySpeedSet ) {
+ audioSink.enqueueData( new AudioSink.AudioFrame(sampleData, data_size, audio_pts ) );
}
}
-
- /** last audio streaming TS */
- private int aSTS = 0;
- /** last video streaming TS */
- private int vSTS = 0;
-
- private long lastAudioTime = 0;
- private static final int audio_dt_d = 400;
- private long lastVideoTime = 0;
- private static final int video_dt_d = 9;
-
+
@Override
- protected final void syncFrame2Audio(TextureFrame frame) {
- /**
- // poor mans video sync .. TODO: off thread 'readNextPackage0(..)' on shared GLContext and multi textures/unit!
- final long now = System.currentTimeMillis();
- // Try sync video to audio
- final long now_d = now - lastAudioTime;
- final long pts_d = vSTS - aSTS - 444; // hack 444 == play video 444ms ahead of audio
- final long dt = Math.min(47, (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ) ;
- //final long dt = (long) ( (float) ( pts_d - now_d ) / getPlaySpeed() ) ;
- final boolean sleep = dt>video_dt_d && dt<1000 && audioSink.getQueuedByteCount()<maxAvailableAudio-10000;
- final long sleepP = dt-video_dt_d;
- if(DEBUG) {
- final int qAT = audioSink.getQueuedTime();
- System.err.println("s: pts-v "+vSTS+", qAT "+qAT+", pts-d "+pts_d+", now_d "+now_d+", dt "+dt+", sleep "+sleep+", sleepP "+sleepP+" ms");
- }
- // ?? Maybe use audioSink.getQueuedTime();
- if( sleep ) {
- try {
- Thread.sleep(sleepP);
- } catch (InterruptedException e) { }
- lastVideoTime = System.currentTimeMillis();
- } else {
- lastVideoTime = now;
- }
- */
- }
+ protected final boolean syncAVRequired() { return true; }
+
private static native int getAvUtilVersion0();
private static native int getAvFormatVersion0();
private static native int getAvCodecVersion0();
@@ -488,10 +503,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
private native void destroyInstance0(long moviePtr);
/**
- * Issues {@link #updateAttributes(int, int, int, int, int, float, int, int, String, String)}
+ * Issues {@link #updateAttributes(int, int, int, int, int, int, int, float, int, int, String, String)}
* and {@link #updateAttributes2(int, int, int, int, int, int, int, int, int, int)}.
+ * <p>
+ * Always uses {@link AudioSink.AudioDataFormat}:
+ * <pre>
+ * [type PCM, sampleRate [10000(?)..44100..48000], sampleSize 16, channelCount 1-2, signed, littleEndian]
+ * </pre>
+ * </p>
*/
- private native void setStream0(long moviePtr, String url, int vid, int aid, int audioFrameCount);
+ private native void setStream0(long moviePtr, String url, int vid, int aid, int audioFrameCount,
+ int aChannelCount, int aSampleRate);
private native void setGLFuncs0(long moviePtr, long procAddrGLTexSubImage2D, long procAddrGLGetError);
private native int getVideoPTS0(long moviePtr);
@@ -505,6 +527,32 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl {
private native int readNextPacket0(long moviePtr, int texTarget, int texFmt, int texType);
private native int seek0(long moviePtr, int position);
+
+ public static enum SampleFormat {
+ // NONE = -1,
+ U8, ///< unsigned 8 bits
+ S16, ///< signed 16 bits
+ S32, ///< signed 32 bits
+ FLT, ///< float
+ DBL, ///< double
+
+ U8P, ///< unsigned 8 bits, planar
+ S16P, ///< signed 16 bits, planar
+ S32P, ///< signed 32 bits, planar
+ FLTP, ///< float, planar
+ DBLP, ///< double, planar
+
+ COUNT; ///< Number of sample formats.
+
+ public static SampleFormat valueOf(int i) {
+ for (SampleFormat fmt : SampleFormat.values()) {
+ if(fmt.ordinal() == i) {
+ return fmt;
+ }
+ }
+ return null;
+ }
+ };
public static enum PixelFormat {
// NONE= -1,
diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java
index a21bb40a8..d03cad28a 100644
--- a/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java
+++ b/src/jogl/classes/jogamp/opengl/util/av/impl/OMXGLMediaPlayer.java
@@ -88,15 +88,16 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl {
@Override
protected void destroyImpl(GL gl) {
- _detachVideoRenderer(moviePtr);
if (moviePtr != 0) {
+ _stop(moviePtr);
+ _detachVideoRenderer(moviePtr);
_destroyInstance(moviePtr);
moviePtr = 0;
}
}
@Override
- protected void initGLStreamImpl(GL gl) throws IOException {
+ protected void initGLStreamImpl(GL gl, int vid, int aid) throws IOException {
if(0==moviePtr) {
throw new GLException("OMX native instance null");
}
@@ -113,12 +114,8 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl {
}
@Override
- protected int getCurrentPositionImpl() {
- return 0!=moviePtr ? _getCurrentPosition(moviePtr) : 0;
- }
- @Override
protected int getAudioPTSImpl() {
- return getCurrentPositionImpl();
+ return 0!=moviePtr ? _getCurrentPosition(moviePtr) : 0;
}
@Override
@@ -131,7 +128,7 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl {
}
@Override
- public synchronized boolean startImpl() {
+ public synchronized boolean playImpl() {
if(0==moviePtr) {
return false;
}
@@ -151,16 +148,6 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl {
/** @return time position after issuing the command */
@Override
- public synchronized boolean stopImpl() {
- if(0==moviePtr) {
- return false;
- }
- _stop(moviePtr);
- return true;
- }
-
- /** @return time position after issuing the command */
- @Override
protected int seekImpl(int msec) {
if(0==moviePtr) {
throw new GLException("OMX native instance null");
@@ -184,8 +171,6 @@ public class OMXGLMediaPlayer extends EGLMediaPlayerImpl {
}
return true;
}
- @Override
- protected void syncFrame2Audio(TextureFrame frame) { }
private String replaceAll(String orig, String search, String repl) {
String dest=null;