From 474ce65081ecd452215bc07ab866666cb11ca8b1 Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Fri, 23 Aug 2013 01:02:33 +0200 Subject: GLMediaPlayer Multithreaded Decoding: GLMediaPlayer* (Part-5) - WIP - Update/fix GLMediaPlayer API doc - GLMediaEventListener: Add event bits for all state changes to be delivered via attributesChanged(..) - StreamWorker / Decoder Thread: - Use StreamWorker only ! - Handle exceptions on StreamWorker via StreamException - Handles stream initialization and decoding (-> initStream(..)) - Split initGLStream(..) -> initStream(..) + initGL(GL) - allow initStream(..)'s implementation being executed on StreamWorker - allow GL initialization to be 'postponed' when stream is read, i.e. non blocking stream initialization (UI .. etc) - Handle EOS via END_OF_STREAM_PTS -> pause/event - Video: Use lock-free LFRingbuffer, similar to ALAudioSink (commit f18a94b3defef16e98badd6d99f2422609aa56c5) +++ - FFMPEGDynamicLibraryBundleInfo - Add avcodec's: - avcodec_get_frame_defaults, avcodec_free_frame (54.28.0), avcodec_flush_buffers, - Add avutil's: - av_frame_unref (55.0.0) - Add avformat's: - avformat_seek_file (??) +++ - FFMPEGMediaPlayer Native: - add 'snoop' video frames for a/v frame count relation. disabled per default, since no more needed due to ALAudioSink's grow-buffer usage of LFRingbuffer. - use sp_avcodec_free_frame if available - 'useRefCountedFrames=1' for libav 55.0 to cache more than one audio frame, not used since ALAudioSink's OpenAL usage does not require it (copies data once). Note: the above snooped-video frame count is used here. - use only one cached audio-frame (-> see above, OpenAL copies data once), while reusing the NIO buffer! - Perform OpenGL sync (glFinish) in native code! - find proper PTS value, i.e. either frame's PTS or DTS, see 'PTSStats'. - FFMPEGMediaPlayer Java: - use private fields - simplified code due to above changes. +++ Working Tests: MovieSimple and MovieCube TODO-1: Fix - Android - OMXGLMediaPlayer TODO-2: - Fix issue where async audio frames arrive much later than 1st video frame, i.e. around 300ms. - Default TextureCount .. maybe 3 ? - Adding Audio synchronization ? - Find 'truth' about correlation of audio and video PTS values, currently, we assume both to be unrelated ? --- .../com/jogamp/opengl/util/av/GLMediaPlayer.java | 210 ++++++++++++++++----- 1 file changed, 162 insertions(+), 48 deletions(-) (limited to 'src/jogl/classes/com/jogamp/opengl') diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java index 512567f33..726eddb01 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java @@ -27,7 +27,6 @@ */ package com.jogamp.opengl.util.av; -import java.io.IOException; import java.net.URI; import javax.media.opengl.GL; @@ -38,22 +37,56 @@ import jogamp.opengl.Debug; import com.jogamp.opengl.util.texture.TextureSequence; /** - * GLMediaPlayer interface specifies a {@link TextureSequence} - * with a video stream as it's source. + * GLMediaPlayer interface specifies a {@link TextureSequence} state machine + * using a multiplexed audio/video stream as it's source. *

- * Audio maybe supported and played back internally or via an {@link AudioSink} implementation, - * if an audio stream is selected in {@link #initGLStream(GL, int, URI, int, int)}. + * Audio maybe supported and played back internally or via an {@link AudioSink} implementation. + *

+ *

+ * Audio and video streams can be selected or muted via {@link #initStream(URI, int, int, int)} + * using the appropriate stream id's. *

* + *
StreamWorker Decoding Thread
 + *

+ * Most of the stream processing is performed on the decoding thread, a.k.a. StreamWorker: + *

+ * StreamWorker generates it's own {@link GLContext}, shared with the one passed to {@link #initGL(GL)}. + * The shared {@link GLContext} allows the decoding thread to push the video frame data directly into + * the designated {@link TextureFrame}, later returned via {@link #getNextTexture(GL)} and used by the user. + *

+ * StreamWorker Error Handling + *

+ * Caught exceptions on StreamWorker are delivered as {@link StreamException}s, + * which either degrades the {@link State} to {@link State#Uninitialized} or {@link State#Paused}. + *

+ *

+ * An occurring {@link StreamException} triggers a {@link GLMediaEventListener#EVENT_CHANGE_ERR EVENT_CHANGE_ERR} event, + * which can be listened to via {@link GLMediaEventListener#attributesChanged(GLMediaPlayer, int, long)}. + *

+ *

+ * An occurred {@link StreamException} can be read via {@link #getStreamException()}. + *

+ * + *

*
GLMediaPlayer Lifecycle
 *

* - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * + * *
action state before state after
{@link #initGLStream(GL, int, URI, int, int)} Uninitialized Paused
{@link #play()} Paused Playing
{@link #pause()} Playing Paused
{@link #seek(int)} Playing, Paused Unchanged
{@link #destroy(GL)} ANY Uninitialized
Action {@link State} Before {@link State} After {@link GLMediaEventListener Event}
{@link #initStream(URI, int, int, int)} {@link State#Uninitialized Uninitialized} {@link State#Initialized Initialized}1, {@link State#Uninitialized Uninitialized} {@link GLMediaEventListener#EVENT_CHANGE_INIT EVENT_CHANGE_INIT} or ( {@link GLMediaEventListener#EVENT_CHANGE_ERR EVENT_CHANGE_ERR} + {@link GLMediaEventListener#EVENT_CHANGE_UNINIT EVENT_CHANGE_UNINIT} )
{@link #initGL(GL)} {@link State#Initialized Initialized} {@link State#Paused Paused}, {@link State#Initialized Initialized} {@link GLMediaEventListener#EVENT_CHANGE_PAUSE EVENT_CHANGE_PAUSE}
{@link #play()} {@link State#Paused Paused} {@link State#Playing Playing} {@link GLMediaEventListener#EVENT_CHANGE_PLAY EVENT_CHANGE_PLAY}
{@link #pause()} {@link State#Playing Playing} {@link State#Paused Paused} {@link GLMediaEventListener#EVENT_CHANGE_PAUSE EVENT_CHANGE_PAUSE}
{@link #seek(int)} {@link State#Paused Paused}, {@link State#Playing Playing} {@link State#Paused Paused}, {@link State#Playing Playing} none
{@link #getNextTexture(GL)} {@link State#Paused Paused}, {@link State#Playing Playing} {@link State#Paused Paused}, {@link State#Playing Playing} none
{@link #getLastTexture()} {@link State#Paused Paused}, {@link State#Playing Playing} {@link State#Paused Paused}, {@link State#Playing Playing} none
{@link TextureFrame#END_OF_STREAM_PTS END_OF_STREAM} {@link State#Playing Playing} {@link State#Paused Paused} {@link GLMediaEventListener#EVENT_CHANGE_EOS EVENT_CHANGE_EOS} + {@link GLMediaEventListener#EVENT_CHANGE_PAUSE EVENT_CHANGE_PAUSE}
{@link StreamException} ANY {@link State#Paused Paused}, {@link State#Uninitialized Uninitialized} {@link GLMediaEventListener#EVENT_CHANGE_ERR EVENT_CHANGE_ERR} + ( {@link GLMediaEventListener#EVENT_CHANGE_PAUSE EVENT_CHANGE_PAUSE} or {@link GLMediaEventListener#EVENT_CHANGE_UNINIT EVENT_CHANGE_UNINIT} )
{@link #destroy(GL)} ANY {@link State#Uninitialized Uninitialized} {@link GLMediaEventListener#EVENT_CHANGE_UNINIT EVENT_CHANGE_UNINIT}
*

* @@ -90,16 +123,18 @@ import com.jogamp.opengl.util.texture.TextureSequence; * will allow tracking time up 2,147,483.647 seconds or * 24 days 20 hours 31 minutes and 23 seconds. * Milliseconds granularity is also more than enough to deal with A-V synchronization, - * where the threshold usually lies within 100ms. + * where the threshold usually lies within 22ms. *

* *
Audio and video synchronization
 *

* The class follows a passive A/V synchronization pattern. - * Audio is being untouched, while {@link #getNextTexture(GL, boolean)} delivers a new video frame - * only, if its timestamp is less than 22ms ahead of time. - * Otherwise the early frame is cached for later retrieval and the previous frame is returned. - * FIXME: Refine! + * Audio is being untouched, while {@link #getNextTexture(GL)} delivers a new video frame + * only, if its timestamp is less than {@link #MAXIMUM_VIDEO_ASYNC} ahead of time. + * If its timestamp is more than {@link #MAXIMUM_VIDEO_ASYNC} ahead of time, + * the previous frame is returned. + * If its timestamp is more than {@link #MAXIMUM_VIDEO_ASYNC} after time, + * the frame is dropped and the next frame is being fetched. *

*

* https://en.wikipedia.org/wiki/Audio_to_video_synchronization @@ -149,37 +184,80 @@ public interface GLMediaPlayer extends TextureSequence { public static final boolean DEBUG = Debug.debug("GLMediaPlayer"); public static final boolean DEBUG_NATIVE = Debug.debug("GLMediaPlayer.Native"); + /** Minimum texture count, value {@value}. */ + public static final int TEXTURE_COUNT_MIN = 4; + /** Constant {@value} for mute or not available. See Audio and video Stream IDs. */ public static final int STREAM_ID_NONE = -2; /** Constant {@value} for auto or unspecified. See Audio and video Stream IDs. */ public static final int STREAM_ID_AUTO = -1; - /** Maximum video frame async .. */ + /** Maximum video frame async of {@value} milliseconds. */ public static final int MAXIMUM_VIDEO_ASYNC = 22; + /** + * A StreamException encapsulates a caught exception in the decoder thread, a.k.a StreamWorker, + * see See StreamWorker Error Handling. + */ + @SuppressWarnings("serial") + public static class StreamException extends Exception { + public StreamException(Throwable cause) { + super(cause); + } + public StreamException(String message, Throwable cause) { + super(message, cause); + } + } public interface GLMediaEventListener extends TexSeqEventListener { - static final int EVENT_CHANGE_VID = 1<<0; - static final int EVENT_CHANGE_AID = 1<<1; - static final int EVENT_CHANGE_SIZE = 1<<2; - static final int EVENT_CHANGE_FPS = 1<<3; - static final int EVENT_CHANGE_BPS = 1<<4; - static final int EVENT_CHANGE_LENGTH = 1<<5; - static final int EVENT_CHANGE_CODEC = 1<<6; + /** State changed to {@link State#Initialized}. See Lifecycle.*/ + static final int EVENT_CHANGE_INIT = 1<<0; + /** State changed to {@link State#Uninitialized}. See Lifecycle.*/ + static final int EVENT_CHANGE_UNINIT = 1<<1; + /** State changed to {@link State#Playing}. See Lifecycle.*/ + static final int EVENT_CHANGE_PLAY = 1<<2; + /** State changed to {@link State#Paused}. See Lifecycle.*/ + static final int EVENT_CHANGE_PAUSE = 1<<3; + /** End of stream reached. See Lifecycle.*/ + static final int EVENT_CHANGE_EOS = 1<<4; + /** An error occurred, e.g. during off-thread initialization. See {@link StreamException} and Lifecycle. */ + static final int EVENT_CHANGE_ERR = 1<<5; + + /** Stream video id change. */ + static final int EVENT_CHANGE_VID = 1<<16; + /** Stream audio id change. */ + static final int EVENT_CHANGE_AID = 1<<17; + /** TextureFrame size change. */ + static final int EVENT_CHANGE_SIZE = 1<<18; + /** Stream fps change. */ + static final int EVENT_CHANGE_FPS = 1<<19; + /** Stream bps change. */ + static final int EVENT_CHANGE_BPS = 1<<20; + /** Stream length change. */ + static final int EVENT_CHANGE_LENGTH = 1<<21; + /** Stream codec change. */ + static final int EVENT_CHANGE_CODEC = 1<<22; /** * @param mp the event source * @param event_mask the changes attributes * @param when system time in msec. */ - public void attributesChanges(GLMediaPlayer mp, int event_mask, long when); + public void attributesChanged(GLMediaPlayer mp, int event_mask, long when); } /** - * See GLMediaPlayer Lifecycle. + * See Lifecycle. */ public enum State { - Uninitialized(0), Playing(1), Paused(2); + /** Uninitialized player, no resources shall be hold. */ + Uninitialized(0), + /** Stream has been initialized, user may play or call {@link #initGL(GL)}. */ + Initialized(1), + /** Stream is playing. */ + Playing(2), + /** Stream is pausing. */ + Paused(3); public final int id; @@ -202,28 +280,62 @@ public interface GLMediaPlayer extends TextureSequence { public void setTextureWrapST(int[] wrapST); /** - * Sets the stream to be used. Initializes all stream related states inclusive OpenGL ones, - * if gl is not null. + * Issues asynchronous stream initialization. + *

+ * Lifecycle: {@link State#Uninitialized} -> {@link State#Initialized}1 or {@link State#Uninitialized} + *

+ *

+ * {@link State#Initialized} is reached asynchronous, + * i.e. user gets notified via {@link GLMediaEventListener#attributesChanged(GLMediaPlayer, int, long) attributesChanges(..)}. + *

+ *

+ * A possible caught asynchronous {@link StreamException} while initializing the stream off-thread + * will be thrown at {@link #initGL(GL)}. + *

+ *

+ * Muted audio can be achieved by passing {@link #STREAM_ID_NONE} to aid. + *

*

- * GLMediaPlayer Lifecycle: Uninitialized -> Paused + * Muted video can be achieved by passing {@link #STREAM_ID_NONE} to vid, + * in which case textureCount is ignored as well as the passed GL object of the subsequent {@link #initGL(GL)} call. *

- * @param gl current GL object. If null, no video output and textures will be available. - * @param textureCount desired number of buffered textures to be decoded off-thread, use 1 for on-thread decoding. * @param streamLoc the stream location * @param vid video stream id, see audio and video Stream IDs * @param aid video stream id, see audio and video Stream IDs - * @return the new state + * @param textureCount desired number of buffered textures to be decoded off-thread, will be validated by implementation. + * The minimum value is {@link #TEXTURE_COUNT_MIN}. + * Ignored if video is muted. + * @throws IllegalStateException if not invoked in {@link State#Uninitialized} + * @throws IllegalArgumentException if arguments are invalid + */ + public void initStream(URI streamLoc, int vid, int aid, int textureCount) throws IllegalStateException, IllegalArgumentException; + + /** + * Returns the {@link StreamException} caught in the decoder thread, or null. + * @see GLMediaEventListener#EVENT_CHANGE_ERR + * @see StreamException + */ + public StreamException getStreamException(); + + /** + * Initializes OpenGL related resources. + *

+ * Lifecycle: {@link State#Initialized} -> {@link State#Paused} or {@link State#Initialized} + *

+ * Argument gl is ignored if video is muted, see {@link #initStream(URI, int, int, int)}. * - * @throws IllegalStateException if not invoked in state Uninitialized - * @throws IOException in case of difficulties to open or process the stream + * @param gl current GL object. Maybe null, for audio only. + * @throws IllegalStateException if not invoked in {@link State#Initialized}. + * @throws IllegalArgumentException if arguments are invalid + * @throws StreamException forwarded from the off-thread stream initialization * @throws GLException in case of difficulties to initialize the GL resources */ - public State initGLStream(GL gl, int textureCount, URI streamLoc, int vid, int aid) throws IllegalStateException, GLException, IOException; + public void initGL(GL gl) throws IllegalStateException, IllegalArgumentException, StreamException, GLException; /** * If implementation uses a {@link AudioSink}, it's instance will be returned. *

- * The {@link AudioSink} instance is available after {@link #initGLStream(GL, int, URI, int, int)}, + * The {@link AudioSink} instance is available after {@link #initStream(URI, int, int, int)}, * if used by implementation. *

*/ @@ -232,7 +344,7 @@ public interface GLMediaPlayer extends TextureSequence { /** * Releases the GL and stream resources. *

- * GLMediaPlayer Lifecycle: ANY -> Uninitialized + * Lifecycle: ANY -> {@link State#Uninitialized} *

*/ public State destroy(GL gl); @@ -250,18 +362,18 @@ public interface GLMediaPlayer extends TextureSequence { public float getPlaySpeed(); /** - * GLMediaPlayer Lifecycle: Paused -> Playing + * Lifecycle: {@link State#Paused} -> {@link State#Playing} */ public State play(); /** - * GLMediaPlayer Lifecycle: Playing -> Paused + * Lifecycle: {@link State#Playing} -> {@link State#Paused} */ public State pause(); /** - * Allowed in state Playing and Paused, otherwise ignored, - * see GLMediaPlayer Lifecycle. + * Allowed in state {@link State#Playing} and {@link State#Paused}, otherwise ignored, + * see Lifecycle. * * @param msec absolute desired time position in milliseconds * @return time current position in milliseconds, after seeking to the desired position @@ -269,8 +381,8 @@ public interface GLMediaPlayer extends TextureSequence { public int seek(int msec); /** - * See GLMediaPlayer Lifecycle. - * @return the current state, either Uninitialized, Playing, Paused + * See Lifecycle. + * @return the current state, either {@link State#Uninitialized}, {@link State#Initialized}, {@link State#Playing} or {@link State#Paused} */ public State getState(); @@ -286,13 +398,13 @@ public interface GLMediaPlayer extends TextureSequence { /** * @return the current decoded frame count since {@link #play()} and {@link #seek(int)} - * as increased by {@link #getNextTexture(GL, boolean)} or the decoding thread. + * as increased by {@link #getNextTexture(GL)} or the decoding thread. */ public int getDecodedFrameCount(); /** * @return the current presented frame count since {@link #play()} and {@link #seek(int)} - * as increased by {@link #getNextTexture(GL, boolean)} for new frames. + * as increased by {@link #getNextTexture(GL)} for new frames. */ public int getPresentedFrameCount(); @@ -311,6 +423,7 @@ public interface GLMediaPlayer extends TextureSequence { *

* See audio and video synchronization. *

+ * @throws IllegalStateException if not invoked in {@link State#Paused} or {@link State#Playing} */ @Override public TextureSequence.TextureFrame getLastTexture() throws IllegalStateException; @@ -324,14 +437,15 @@ public interface GLMediaPlayer extends TextureSequence { *

* See audio and video synchronization. *

+ * @throws IllegalStateException if not invoked in {@link State#Paused} or {@link State#Playing} * * @see #addEventListener(GLMediaEventListener) * @see GLMediaEventListener#newFrameAvailable(GLMediaPlayer, TextureFrame, long) */ @Override - public TextureSequence.TextureFrame getNextTexture(GL gl, boolean blocking) throws IllegalStateException; + public TextureSequence.TextureFrame getNextTexture(GL gl) throws IllegalStateException; - /** Return the stream location, as set by {@link #initGLStream(GL, int, URI, int, int)}. */ + /** Return the stream location, as set by {@link #initStream(URI, int, int, int)}. */ public URI getURI(); /** -- cgit v1.2.3