From 6c67d73dc6b9e49bdd406902e533be91db1a3b0a Mon Sep 17 00:00:00 2001 From: Sven Göthel Date: Sun, 28 Jan 2024 06:42:50 +0100 Subject: GLMediaPlayer/FFMPEGMediaPlayer: Add working subtitle (text + ass/saa) support via FFMpeg TODO: - We may want to refine subtitle PTS handling - We may want to support bitmapped subtitles --- .../com/jogamp/opengl/util/av/ASSEventLine.java | 135 ++++++++++++++++++ .../jogamp/opengl/util/av/ASSEventListener.java | 35 +++++ .../com/jogamp/opengl/util/av/GLMediaPlayer.java | 5 + .../android/av/AndroidGLMediaPlayerAPI14.java | 22 +-- .../jogamp/opengl/util/av/GLMediaPlayerImpl.java | 34 +++-- .../jogamp/opengl/util/av/NullGLMediaPlayer.java | 10 +- .../av/impl/FFMPEGDynamicLibraryBundleInfo.java | 14 +- .../opengl/util/av/impl/FFMPEGMediaPlayer.java | 14 ++ src/jogl/native/libav/ffmpeg_impl_template.c | 153 +++++++++++++++++++-- src/jogl/native/libav/ffmpeg_static.c | 8 +- src/jogl/native/libav/ffmpeg_static.h | 2 + src/jogl/native/libav/ffmpeg_tool.h | 5 + 12 files changed, 401 insertions(+), 36 deletions(-) create mode 100644 src/jogl/classes/com/jogamp/opengl/util/av/ASSEventLine.java create mode 100644 src/jogl/classes/com/jogamp/opengl/util/av/ASSEventListener.java diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventLine.java b/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventLine.java new file mode 100644 index 000000000..6389df773 --- /dev/null +++ b/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventLine.java @@ -0,0 +1,135 @@ +/** + * Copyright 2024 JogAmp Community. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY JogAmp Community ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JogAmp Community OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are those of the + * authors and should not be interpreted as representing official policies, either expressed + * or implied, of JogAmp Community. + */ +package com.jogamp.opengl.util.av; + +/** + * ASS/SAA Event Line of subtitles + *

+ * See http://www.tcax.org/docs/ass-specs.htm + *

+ */ +public class ASSEventLine { + public enum Format { + UNKNOWN, + /** FFMpeg output w/o start, end: + * 
+           0    1      2      3     4        5        6        7       8
+           Seq, Layer, Style, Name, MarginL, MarginR, MarginV, Effect, TEXT
+         *
+ */ + FFMPEG, + /** Just the plain text part */ + TEXT + }; + public final Format source; + public final int seqnr; + public final int pts_start; + public final int pts_end; + public final int layer; + public final String style; + public final String name; + /** Actual subtitle text */ + public final String text; + /** Number of lines of {@link #text}, i.e. occurrence of {@code \n} + 1. */ + public final int lines; + + /** + * ASS/SAA Event Line ctor + * @param fmt input format of {@code ass}, currently only {@link ASSEventLine.Format#FFMPEG} and {@link ASSEventLine.Format#TEXT} is supported + * @param ass ASS/SAA compatible event line according to {@code fmt} + * @param pts_start pts start in ms, provided for {@link ASSEventLine.Format#FFMPEG} and {@link ASSEventLine.Format#TEXT} + * @param pts_end pts end in ms, provided for {@link ASSEventLine.Format#FFMPEG} and {@link ASSEventLine.Format#TEXT} + */ + public ASSEventLine(final Format fmt, final String ass, final int pts_start, final int pts_end) { + this.source = fmt; + int seqnr = 0; + int layer = 0; + String style = "Default"; + String name = ""; + String text = ""; + if( Format.FFMPEG == fmt ) { + final int len = null != ass ? ass.length() : 0; + int part = 0; + for(int i=0; 9 > part && len > i; ) { + if( 8 == part ) { + text = ass.substring(i); + } else { + final int j = ass.indexOf(',', i); + if( 0 > j ) { + break; + } + final String v = ass.substring(i, j); + switch(part) { + case 0: + seqnr = Integer.valueOf(v); + break; + case 1: + layer = Integer.valueOf(v); + break; + case 2: + style = v; + break; + case 3: + name = v; + break; + } + i = j + 1; + } + ++part; + } + } else if( Format.TEXT == fmt ) { + text = ass; + } + this.seqnr = seqnr; + this.pts_start = pts_start; + this.pts_end = pts_end; + this.layer = layer; + this.style = style; + this.name = name; + this.text = text.replace("\\N", "\n"); + { + final int len = this.text.length(); + int lc = 1; + for(int i=0; len > i; ) { + final int j = this.text.indexOf("\n", i); + if( 0 > j ) { + break; + } + ++lc; + i = j + 1; + } + this.lines = lc; + } + } + public int getDuration() { return pts_end - pts_start + 1; } + + @Override + public String toString() { + return "ASS["+source+", #"+seqnr+", l_"+layer+", ["+pts_start+".."+pts_end+"] "+getDuration()+" ms, style "+style+", name '"+name+"': '"+text+"' ("+lines+")]"; + } +} diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventListener.java b/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventListener.java new file mode 100644 index 000000000..3baf6b69c --- /dev/null +++ b/src/jogl/classes/com/jogamp/opengl/util/av/ASSEventListener.java @@ -0,0 +1,35 @@ +/** + * Copyright 2024 JogAmp Community. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY JogAmp Community ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JogAmp Community OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are those of the + * authors and should not be interpreted as representing official policies, either expressed + * or implied, of JogAmp Community. + */ +package com.jogamp.opengl.util.av; + +/** + * {@link ASSEventLine} Listener + */ +public interface ASSEventListener { + void run(ASSEventLine e); +} diff --git a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java index 646280f08..698504205 100644 --- a/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java +++ b/src/jogl/classes/com/jogamp/opengl/util/av/GLMediaPlayer.java @@ -899,6 +899,11 @@ public interface GLMediaPlayer extends TextureSequence { /** Return all {@link GLMediaEventListener} of this player. */ public GLMediaEventListener[] getEventListeners(); + /** Sets the {@link ASSEventListener} for this player. */ + public void setASSEventListener(ASSEventListener l); + /** Returns the {@link #setASSEventListener(ASSEventListener)} of this player. */ + public ASSEventListener getASSEventListener(); + /** * Returns the attached user object for the given name. */ diff --git a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java index 3585f7ab2..c4d1ee78f 100644 --- a/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java +++ b/src/jogl/classes/jogamp/opengl/android/av/AndroidGLMediaPlayerAPI14.java @@ -281,7 +281,7 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } @Override - protected final void initStreamImpl(final int vid, final int aid, int sid) throws IOException { + protected final void initStreamImpl(final int vid, final int aid, final int sid) throws IOException { if( null == getUri() ) { return; } @@ -336,10 +336,12 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { r_alangs = new String[] { "n/a" }; } final String icodec = "android"; - updateAttributes(null, new int[] { 0 }, new String[] { "und" }, - 0 /* fake */, r_aids, r_alangs, - r_aid, new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, mp.getVideoWidth(), mp.getVideoHeight(), 0, 0, 0, 0f, 0, 0, mp.getDuration(), icodec, icodec); + updateAttributes(null, + new int[] { 0 }, new String[] { "und" }, 0 /* fake */, + r_aids, r_alangs, r_aid, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + mp.getVideoWidth(), mp.getVideoHeight(), 0, 0, 0, 0f, 0, 0, mp.getDuration(), + icodec, icodec, null); /** mp.setOnPreparedListener(new MediaPlayer.OnPreparedListener() { @Override @@ -372,10 +374,12 @@ public class AndroidGLMediaPlayerAPI14 extends GLMediaPlayerImpl { } } } - updateAttributes(null, new int[]{0}, new String[] { "und" }, - 0 /* fake */, new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, size.width, size.height, 0, 0, 0, fpsRange[1]/1000f, 0, 0, 0, icodec, icodec); + updateAttributes(null, + new int[]{0}, new String[] { "und" }, 0 /* fake */, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + size.width, size.height, 0, 0, 0, fpsRange[1]/1000f, 0, 0, 0, + icodec, icodec, null); } } private static String camSz2Str(final Camera.Size csize) { diff --git a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java index 5df858b2d..a36213a01 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java +++ b/src/jogl/classes/jogamp/opengl/util/av/GLMediaPlayerImpl.java @@ -66,6 +66,7 @@ import com.jogamp.common.util.TSPrinter; import com.jogamp.common.util.WorkerThread; import com.jogamp.math.FloatUtil; import com.jogamp.opengl.GLExtensions; +import com.jogamp.opengl.util.av.ASSEventListener; import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.glsl.ShaderCode; import com.jogamp.opengl.util.texture.Texture; @@ -175,6 +176,8 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { private String acodec = unknown; /** Shall be set by the {@link #initStreamImpl(int, int, int)} method implementation. */ private String vcodec = unknown; + /** Shall be set by the {@link #initStreamImpl(int, int, int)} method implementation. */ + private String scodec = unknown; private volatile int decodedFrameCount = 0; private int presentedFrameCount = 0; @@ -196,6 +199,8 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { protected AudioSink audioSink = null; protected boolean audioSinkPlaySpeedSet = false; + protected volatile ASSEventListener assEventListener = null; + /** AV System Clock Reference (SCR) */ private final PTS av_scr = new PTS( () -> { return State.Playing == state ? playSpeed : 0f; } ); private final PTS av_scr_cpy = new PTS( av_scr ); @@ -952,11 +957,12 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { final float _fps = 24f; final int _duration = 10*60*1000; // msec final int _totalFrames = (int) ( (_duration/1000)*_fps ); - updateAttributes("test", new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, new int[0], new String[0], // audio - GLMediaPlayer.STREAM_ID_NONE, new int[0], new String[0], // subs - GLMediaPlayer.STREAM_ID_NONE, - TestTexture.singleton.getWidth(), TestTexture.singleton.getHeight(), 0, 0, 0, _fps, _totalFrames, 0, _duration, "png-static", null); + updateAttributes("test", + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + TestTexture.singleton.getWidth(), TestTexture.singleton.getHeight(), 0, 0, 0, _fps, _totalFrames, 0, _duration, + "png-static", null, null); } protected abstract TextureFrame createTexImage(GL gl, int texName); @@ -1704,7 +1710,9 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { int vid, final int[] a_streams, final String[] a_langs, int aid, final int[] s_streams, final String[] s_langs, int sid, final int width, final int height, - final int bps_stream, final int bps_video, final int bps_audio, final float fps, final int videoFrames, final int audioFrames, final int duration, final String vcodec, final String acodec) { + final int bps_stream, final int bps_video, final int bps_audio, + final float fps, final int videoFrames, final int audioFrames, final int duration, + final String vcodec, final String acodec, final String scodec) { final GLMediaPlayer.EventMask eventMask = new GLMediaPlayer.EventMask(); final boolean wasUninitialized = state == State.Uninitialized; @@ -1803,6 +1811,10 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { eventMask.setBit(GLMediaPlayer.EventMask.Bit.Codec); this.vcodec = vcodec; } + if( (null!=scodec && scodec.length()>0 && !this.scodec.equals(scodec)) ) { + eventMask.setBit(GLMediaPlayer.EventMask.Bit.Codec); + this.scodec = scodec; + } if( eventMask.isZero() ) { return; } @@ -1992,7 +2004,7 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { ", Texture[count "+textureCount+", free "+freeVideoFrames+", dec "+decVideoFrames+", tagt "+toHexString(textureTarget)+", ifmt "+toHexString(textureInternalFormat)+", fmt "+toHexString(textureFormat)+", type "+toHexString(textureType)+"], "+ "Video[id "+vid+"/"+Arrays.toString(v_streams)+"/"+Arrays.toString(v_langs)+", <"+vcodec+">, "+width+"x"+height+", glOrient "+isInGLOrientation+", "+fps+" fps, "+frame_duration+" fdur, "+bps_video+" bps], "+ "Audio[id "+aid+"/"+Arrays.toString(a_streams)+"/"+Arrays.toString(a_langs)+", <"+acodec+">, "+bps_audio+" bps, "+audioFrames+" frames], "+ - "Subs[id "+sid+"/"+Arrays.toString(s_streams)+"/"+Arrays.toString(s_langs)+"], uri "+loc+camPath+"]"; + "Subs[id "+sid+"/"+Arrays.toString(s_streams)+"/"+Arrays.toString(s_langs)+", <"+scodec+">], uri "+loc+camPath+"]"; } @Override @@ -2066,9 +2078,15 @@ public abstract class GLMediaPlayerImpl implements GLMediaPlayer { return eventListeners.toArray(new GLMediaEventListener[eventListeners.size()]); } } - private final Object eventListenersLock = new Object(); + @Override + public final void setASSEventListener(final ASSEventListener l) { + this.assEventListener = l; + } + @Override + public final ASSEventListener getASSEventListener() { return assEventListener; } + @Override public final Object getAttachedObject(final String name) { return attachedObjects.get(name); diff --git a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java index f0f06bf2a..44031372f 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/NullGLMediaPlayer.java @@ -149,10 +149,12 @@ public class NullGLMediaPlayer extends GLMediaPlayerImpl { final float _fps = 24f; final int _duration = 10*60*1000; // msec final int _totalFrames = (int) ( (_duration/1000)*_fps ); - updateAttributes("null", new int[] { 0 }, new String[] { "und" }, - 0 /* fake */, new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, new int[0], new String[0], - GLMediaPlayer.STREAM_ID_NONE, texData.getWidth(), texData.getHeight(), 0, 0, 0, _fps, _totalFrames, 0, _duration, "png-static", null); + updateAttributes("null", + new int[] { 0 }, new String[] { "und" }, 0 /* fake */, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + new int[0], new String[0], GLMediaPlayer.STREAM_ID_NONE, + texData.getWidth(), texData.getHeight(), 0, 0, 0, _fps, _totalFrames, 0, _duration, + "png-static", null, null); } @Override protected final void initGLImpl(final GL gl) throws IOException, GLException { diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java index 05a0ddb64..cfe0f72af 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java @@ -52,7 +52,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { private static final List glueLibNames = new ArrayList(); // none - private static final int symbolCount = 61; + private static final int symbolCount = 63; private static final String[] symbolNames = { "avutil_version", "avformat_version", @@ -80,7 +80,9 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "av_packet_unref", "avcodec_send_packet", // 57 "avcodec_receive_frame", // 57 - /* +18 = 23 */ + "avcodec_decode_subtitle2", // 52.23.0 + "avsubtitle_free", // 52.82.0 + /* +20 = 25 */ // libavutil "av_pix_fmt_desc_get", // >= lavu 51.45 @@ -100,7 +102,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "av_channel_layout_uninit", // >= 59 (opt) "av_channel_layout_describe", // >= 59 (opt) "av_opt_set_chlayout", // >= 59 - /* +16 = 40 */ + /* +17 = 42 */ // libavformat "avformat_alloc_context", @@ -117,11 +119,11 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "avformat_network_init", // 53.13.0 (opt) "avformat_network_deinit", // 53.13.0 (opt) "avformat_find_stream_info", // 53.3.0 (opt) - /* +14 = 54 */ + /* +14 = 56 */ // libavdevice "avdevice_register_all", // supported in all versions (opt) - /* +1 = 55 */ + /* +1 = 57 */ // libswresample "av_opt_set_sample_fmt", // actually lavu .. but exist only w/ swresample! @@ -130,7 +132,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "swr_free", "swr_convert", "swr_get_out_samples", - /* +6 = 61 */ + /* +6 = 63 */ }; // optional symbol names diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index 464b8c29d..0cc36cc91 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -40,11 +40,13 @@ import com.jogamp.common.av.AudioFormat; import com.jogamp.common.av.AudioSink; import com.jogamp.common.av.AudioSinkFactory; import com.jogamp.common.av.TimeFrameI; +import com.jogamp.common.os.Clock; import com.jogamp.common.util.IOUtil; import com.jogamp.common.util.PropertyAccess; import com.jogamp.common.util.SecurityUtil; import com.jogamp.gluegen.runtime.ProcAddressTable; import com.jogamp.opengl.util.GLPixelStorageModes; +import com.jogamp.opengl.util.av.ASSEventLine; import com.jogamp.opengl.util.av.GLMediaPlayer; import com.jogamp.opengl.util.texture.Texture; @@ -999,5 +1001,17 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { audioSink.enqueueData( audio_pts, sampleData, data_size); } } + final void pushSubtitleText(final String text, final int pts, final int start_display_pts, final int end_display_pts) { + if( null != assEventListener ) { + if( start_display_pts > getPTS().get(Clock.currentMillis()) ) { + assEventListener.run( new ASSEventLine(ASSEventLine.Format.TEXT, text, start_display_pts, end_display_pts) ); + } + } + } + final void pushSubtitleASS(final String ass, final int pts, final int start_display_pts, final int end_display_pts) { + if( null != assEventListener ) { + assEventListener.run( new ASSEventLine(ASSEventLine.Format.FFMPEG, ass, start_display_pts, end_display_pts) ); + } + } } diff --git a/src/jogl/native/libav/ffmpeg_impl_template.c b/src/jogl/native/libav/ffmpeg_impl_template.c index 1d938102f..e3a1590f4 100644 --- a/src/jogl/native/libav/ffmpeg_impl_template.c +++ b/src/jogl/native/libav/ffmpeg_impl_template.c @@ -69,6 +69,8 @@ typedef int (APIENTRYP AV_NEW_PACKET)(AVPacket *pkt, int size); typedef void (APIENTRYP AV_PACKET_UNREF)(AVPacket *pkt); typedef int (APIENTRYP AVCODEC_SEND_PACKET)(AVCodecContext *avctx, AVPacket *avpkt); // 57 typedef int (APIENTRYP AVCODEC_RECEIVE_FRAME)(AVCodecContext *avctx, AVFrame *picture); // 57 +typedef int (APIENTRYP AVCODEC_DECODE_SUBTITLE2)(AVCodecContext *avctx, AVSubtitle *sub, int *got_sub_ptr, const AVPacket *avpkt); // 52.23 +typedef int (APIENTRYP AV_SUBTITLE_FREE)(AVSubtitle *sub); // 52.82 static AVCODEC_CLOSE sp_avcodec_close; static AVCODEC_STRING sp_avcodec_string; @@ -89,7 +91,9 @@ static AV_PACKET_UNREF sp_av_packet_unref; static AVCODEC_SEND_PACKET sp_avcodec_send_packet; // 57 static AVCODEC_RECEIVE_FRAME sp_avcodec_receive_frame; // 57 -// count: +18 = 23 +static AVCODEC_DECODE_SUBTITLE2 sp_avcodec_decode_subtitle2; // 52.23 +static AV_SUBTITLE_FREE sp_avsubtitle_free; // 52.82 +// count: +20 = 25 // libavutil typedef AVPixFmtDescriptor* (APIENTRYP AV_PIX_FMT_DESC_GET)(enum AVPixelFormat pix_fmt); // lavu >= 51.45; lavu 51: 'enum PixelFormat pix_fmt', lavu 53: 'enum AVPixelFormat pix_fmt' @@ -127,7 +131,7 @@ static AV_CHANNEL_LAYOUT_DEFAULT sp_av_channel_layout_default; // >= 59 static AV_CHANNEL_LAYOUT_UNINIT sp_av_channel_layout_uninit; // >= 59 static AV_CHANNEL_LAYOUT_DESCRIBE sp_av_channel_layout_describe; // >= 59 static AV_OPT_SET_CHLAYOUT sp_av_opt_set_chlayout; // >= 59 -// count: +17 = 40 +// count: +17 = 42 // libavformat typedef AVFormatContext *(APIENTRYP AVFORMAT_ALLOC_CONTEXT)(void); @@ -159,12 +163,12 @@ static AV_READ_PAUSE sp_av_read_pause; static AVFORMAT_NETWORK_INIT sp_avformat_network_init; // 53.13.0 static AVFORMAT_NETWORK_DEINIT sp_avformat_network_deinit; // 53.13.0 static AVFORMAT_FIND_STREAM_INFO sp_avformat_find_stream_info; // 53.3.0 -// count: +14 = 54 +// count: +14 = 56 // libavdevice [53.0.0] typedef int (APIENTRYP AVDEVICE_REGISTER_ALL)(void); static AVDEVICE_REGISTER_ALL sp_avdevice_register_all; -// count: +1 = 55 +// count: +1 = 57 // libswresample [1...] typedef int (APIENTRYP AV_OPT_SET_SAMPLE_FMT)(void *obj, const char *name, enum AVSampleFormat fmt, int search_flags); // actually lavu .. but exist only w/ swresample! @@ -180,7 +184,7 @@ static SWR_INIT sp_swr_init; static SWR_FREE sp_swr_free; static SWR_CONVERT sp_swr_convert; static SWR_GET_OUT_SAMPLES sp_swr_get_out_samples; -// count: +6 = 61 +// count: +6 = 66 static const char * const ClazzNameString = "java/lang/String"; @@ -206,7 +210,7 @@ static const char * const ClazzNameString = "java/lang/String"; #define MY_MUTEX_UNLOCK(e,s) #endif -#define SYMBOL_COUNT 61 +#define SYMBOL_COUNT 63 JNIEXPORT jboolean JNICALL FF_FUNC(initSymbols0) (JNIEnv *env, jobject instance, jobject jmutex_avcodec_openclose, jobject jSymbols, jint count) @@ -251,6 +255,8 @@ JNIEXPORT jboolean JNICALL FF_FUNC(initSymbols0) sp_av_packet_unref = (AV_PACKET_UNREF) (intptr_t) symbols[i++]; sp_avcodec_send_packet = (AVCODEC_SEND_PACKET) (intptr_t) symbols[i++]; sp_avcodec_receive_frame = (AVCODEC_RECEIVE_FRAME) (intptr_t) symbols[i++]; + sp_avcodec_decode_subtitle2 = (AVCODEC_DECODE_SUBTITLE2) (intptr_t) symbols[i++]; + sp_avsubtitle_free = (AV_SUBTITLE_FREE) (intptr_t) symbols[i++]; sp_av_pix_fmt_desc_get = (AV_PIX_FMT_DESC_GET) (intptr_t) symbols[i++]; sp_av_frame_unref = (AV_FRAME_UNREF) (intptr_t) symbols[i++]; @@ -458,7 +464,8 @@ static void _updateJavaAttributes(JNIEnv *env, FFMPEGToolBasicAV_t* pAV) { pAV->bps_stream, pAV->bps_video, pAV->bps_audio, pAV->fps, pAV->frames_video, pAV->frames_audio, pAV->duration, (*env)->NewStringUTF(env, pAV->vcodec), - (*env)->NewStringUTF(env, pAV->acodec) ); + (*env)->NewStringUTF(env, pAV->acodec), + (*env)->NewStringUTF(env, pAV->scodec)); JoglCommon_ExceptionCheck1_throwNewRuntimeException(env, "FFmpeg: Exception occured at updateAttributes(..)"); } } @@ -491,6 +498,14 @@ static void freeInstance(JNIEnv *env, FFMPEGToolBasicAV_t* pAV) { } pAV->pACodec=NULL; + // Close the S codec + if(NULL != pAV->pSCodecCtx) { + sp_avcodec_close(pAV->pSCodecCtx); + sp_avcodec_free_context(&pAV->pSCodecCtx); + pAV->pSCodecCtx = NULL; + } + pAV->pSCodec=NULL; + // Close the video file if(NULL != pAV->pFormatCtx) { sp_avformat_close_input(&pAV->pFormatCtx); @@ -1018,6 +1033,7 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0) } // Customize .. + pAV->pACodecCtx->pkt_timebase = pAV->pAStream->time_base; // pAV->pACodecCtx->thread_count=2; // pAV->pACodecCtx->thread_type=FF_THREAD_FRAME|FF_THREAD_SLICE; // Decode more than one frame at once pAV->pACodecCtx->thread_count=0; @@ -1028,7 +1044,6 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0) // Note: OpenAL well supports n-channel by now (SOFT), // however - AFAIK AV_SAMPLE_FMT_S16 would allow no conversion! pAV->pACodecCtx->request_sample_fmt=AV_SAMPLE_FMT_S16; - pAV->pACodecCtx->skip_frame=AVDISCARD_DEFAULT; sp_avcodec_string(pAV->acodec, sizeof(pAV->acodec), pAV->pACodecCtx, 0); @@ -1209,6 +1224,7 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0) return; } // Customize .. + pAV->pVCodecCtx->pkt_timebase = pAV->pVStream->time_base; // pAV->pVCodecCtx->thread_count=2; // pAV->pVCodecCtx->thread_type=FF_THREAD_FRAME|FF_THREAD_SLICE; // Decode more than one frame at once pAV->pVCodecCtx->thread_count=0; @@ -1314,8 +1330,56 @@ JNIEXPORT void JNICALL FF_FUNC(setStream0) } sp_av_frame_unref(pAV->pVFrame); } + + if(0<=pAV->sid) { + // Get a pointer to the codec context for the video stream + // FIXME: Libav Binary compatibility! JAU01 + pAV->pSCodecPar = pAV->pSStream->codecpar; + #if 0 + pAV->pSCodecCtx->get_format = my_get_format; + #endif + + // Find the decoder for the video stream + pAV->pSCodec=sp_avcodec_find_decoder(pAV->pSCodecPar->codec_id); + if(pAV->pSCodec==NULL) { + JoglCommon_throwNewRuntimeException(env, "Couldn't find subtitle codec for codec_id %d", pAV->pSCodecPar->codec_id); + return; + } + + // Allocate the decoder context for the video stream + pAV->pSCodecCtx = sp_avcodec_alloc_context3(pAV->pSCodec); + if(pAV->pSCodecCtx==NULL) { + JoglCommon_throwNewRuntimeException(env, "Couldn't allocate subtitle decoder context for codec_id %d", pAV->pSCodecPar->codec_id); + return; + } + res = sp_avcodec_parameters_to_context(pAV->pSCodecCtx, pAV->pSCodecPar); + if(res<0) { + JoglCommon_throwNewRuntimeException(env, "Couldn't copy video codec-par to context"); + return; + } + // Customize .. + pAV->pSCodecCtx->pkt_timebase = pAV->pSStream->time_base; + pAV->pVCodecCtx->thread_count=0; + pAV->pVCodecCtx->thread_type=0; + pAV->pVCodecCtx->workaround_bugs=FF_BUG_AUTODETECT; + pAV->pVCodecCtx->skip_frame=AVDISCARD_DEFAULT; + + sp_avcodec_string(pAV->scodec, sizeof(pAV->scodec), pAV->pSCodecCtx, 0); + + // Open codec + MY_MUTEX_LOCK(env, mutex_avcodec_openclose); + { + res = sp_avcodec_open2(pAV->pSCodecCtx, pAV->pSCodec, NULL); + } + MY_MUTEX_UNLOCK(env, mutex_avcodec_openclose); + if(res<0) { + JoglCommon_throwNewRuntimeException(env, "Couldn't open subtitle codec %d, %s", pAV->pSCodecCtx->codec_id, pAV->scodec); + return; + } + } pAV->vPTS=0; pAV->aPTS=0; + pAV->sPTS=0; initPTSStats(&pAV->vPTSStats); initPTSStats(&pAV->aPTSStats); pAV->ready = 1; @@ -1711,6 +1775,76 @@ JNIEXPORT jint JNICALL FF_FUNC(readNextPacket0) sp_av_frame_unref(pAV->pVFrame); } // draining frames loop + } else if(stream_id == pAV->sid) { + // Decode Subtitle package + int res = 0; + int got_sub = 0, got_sub2 = 0; + AVSubtitle sub; + + res = sp_avcodec_decode_subtitle2(pAV->pSCodecCtx, &sub, &got_sub, pAV->packet); + if (0 > res) { + res = 0; + if( pAV->verbose ) { + fprintf(stderr, "S-P: EOF.0\n"); + } + } else { + // OK + if( !got_sub ) { + if( pAV->packet->data ) { + // EAGAIN + } else { + // EOF + if( pAV->verbose ) { + fprintf(stderr, "S-P: EOF.1\n"); + } + } + } else { + if (!pAV->packet->data) { + // .. pending .. + if( pAV->verbose ) { + fprintf(stderr, "S-P: Pending\n"); + } + } else { + got_sub2 = 1; + } + } + } + if( got_sub2 ) { + int32_t sPTS, sStart, sEnd; + if( AV_NOPTS_VALUE == sub.pts ) { + sPTS = -1; + sStart = -1; + sEnd = -1; + } else { + sPTS = my_av_q2i32( sub.pts * 1000, AV_TIME_BASE_Q); + sStart = my_av_q2i32( ( sub.pts + sub.start_display_time ) * 1000, AV_TIME_BASE_Q); + sEnd = my_av_q2i32( ( sub.pts + sub.end_display_time ) * 1000, AV_TIME_BASE_Q); + } + for(unsigned int i=0; itype && NULL != r->text ) { + if( pAV->verbose ) { + fprintf(stderr, "S[f %d, i %d, pts %d[%d..%d]]: %s\n", (int)r->type, i, r->text, sPTS, sStart, sEnd); + } + (*env)->CallVoidMethod(env, pAV->ffmpegMediaPlayer, ffmpeg_jni_mid_pushSubtitleText, (*env)->NewStringUTF(env, r->text), sPTS, sStart, sEnd); + JoglCommon_ExceptionCheck1_throwNewRuntimeException(env, "FFmpeg: Exception occured at pushSubtitleText(..)"); + } else if( SUBTITLE_ASS == r->type && NULL != r->ass ) { + if( pAV->verbose ) { + fprintf(stderr, "S[f %d, i %d, pts %d[%d..%d]]: %s\n", (int)r->type, i, r->ass, sPTS, sStart, sEnd); + } + (*env)->CallVoidMethod(env, pAV->ffmpegMediaPlayer, ffmpeg_jni_mid_pushSubtitleASS, (*env)->NewStringUTF(env, r->ass), sPTS, sStart, sEnd); + JoglCommon_ExceptionCheck1_throwNewRuntimeException(env, "FFmpeg: Exception occured at pushSubtitleASS(..)"); + } else { + if( pAV->verbose ) { + fprintf(stderr, "S[f %d, i %d]: null\n", (int)r->type, i); + } + } + } + pAV->sPTS = sPTS; + } + if( got_sub ) { + sp_avsubtitle_free(&sub); + } } // stream_id selection sp_av_packet_unref(pAV->packet); } @@ -1817,6 +1951,9 @@ JNIEXPORT jint JNICALL FF_FUNC(seek0) if(NULL != pAV->pACodecCtx) { sp_avcodec_flush_buffers( pAV->pACodecCtx ); } + if(NULL != pAV->pSCodecCtx) { + sp_avcodec_flush_buffers( pAV->pSCodecCtx ); + } const jint rPTS = my_av_q2i32( ( pAV->vid >= 0 ? pAV->pVFrame->pts : pAV->pAFrames[pAV->aFrameCurrent]->pts ) * 1000, time_base); if(pAV->verbose) { fprintf(stderr, "SEEK: post : res %d, u %d\n", res, rPTS); diff --git a/src/jogl/native/libav/ffmpeg_static.c b/src/jogl/native/libav/ffmpeg_static.c index 10b388e75..184972306 100644 --- a/src/jogl/native/libav/ffmpeg_static.c +++ b/src/jogl/native/libav/ffmpeg_static.c @@ -36,6 +36,8 @@ static const char * const ClazzNameFFMPEGMediaPlayer = "jogamp/opengl/util/av/im static jclass ffmpegMediaPlayerClazz = NULL; jmethodID ffmpeg_jni_mid_pushSound = NULL; +jmethodID ffmpeg_jni_mid_pushSubtitleText = NULL; +jmethodID ffmpeg_jni_mid_pushSubtitleASS = NULL; jmethodID ffmpeg_jni_mid_updateAttributes = NULL; jmethodID ffmpeg_jni_mid_setIsGLOriented = NULL; jmethodID ffmpeg_jni_mid_setupFFAttributes = NULL; @@ -65,13 +67,17 @@ JNIEXPORT jboolean JNICALL Java_jogamp_opengl_util_av_impl_FFMPEGStaticNatives_i } ffmpeg_jni_mid_pushSound = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "pushSound", "(Ljava/nio/ByteBuffer;II)V"); + ffmpeg_jni_mid_pushSubtitleText = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "pushSubtitleText", "(Ljava/lang/String;III)V"); + ffmpeg_jni_mid_pushSubtitleASS = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "pushSubtitleASS", "(Ljava/lang/String;III)V"); ffmpeg_jni_mid_updateAttributes = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "updateAttributes", - "(Ljava/lang/String;[I[Ljava/lang/String;I[I[Ljava/lang/String;I[I[Ljava/lang/String;IIIIIIFIIILjava/lang/String;Ljava/lang/String;)V"); + "(Ljava/lang/String;[I[Ljava/lang/String;I[I[Ljava/lang/String;I[I[Ljava/lang/String;IIIIIIFIIILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); ffmpeg_jni_mid_setIsGLOriented = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "setIsGLOriented", "(Z)V"); ffmpeg_jni_mid_setupFFAttributes = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "setupFFAttributes", "(IIIIIIIIIIIIIII)V"); ffmpeg_jni_mid_isAudioFormatSupported = (*env)->GetMethodID(env, ffmpegMediaPlayerClazz, "isAudioFormatSupported", "(III)Z"); if(ffmpeg_jni_mid_pushSound == NULL || + ffmpeg_jni_mid_pushSubtitleText == NULL || + ffmpeg_jni_mid_pushSubtitleASS == NULL || ffmpeg_jni_mid_updateAttributes == NULL || ffmpeg_jni_mid_setIsGLOriented == NULL || ffmpeg_jni_mid_setupFFAttributes == NULL || diff --git a/src/jogl/native/libav/ffmpeg_static.h b/src/jogl/native/libav/ffmpeg_static.h index ab4db2506..c77783424 100644 --- a/src/jogl/native/libav/ffmpeg_static.h +++ b/src/jogl/native/libav/ffmpeg_static.h @@ -41,6 +41,8 @@ #include "jogamp_opengl_util_av_impl_FFMPEGStaticNatives.h" extern jmethodID ffmpeg_jni_mid_pushSound; +extern jmethodID ffmpeg_jni_mid_pushSubtitleText; +extern jmethodID ffmpeg_jni_mid_pushSubtitleASS; extern jmethodID ffmpeg_jni_mid_updateAttributes; extern jmethodID ffmpeg_jni_mid_setIsGLOriented; extern jmethodID ffmpeg_jni_mid_setupFFAttributes; diff --git a/src/jogl/native/libav/ffmpeg_tool.h b/src/jogl/native/libav/ffmpeg_tool.h index 5e23ca882..852ff8d1e 100644 --- a/src/jogl/native/libav/ffmpeg_tool.h +++ b/src/jogl/native/libav/ffmpeg_tool.h @@ -200,6 +200,10 @@ typedef struct { int32_t s_streams[MAX_STREAM_COUNT]; int32_t sid; AVStream* pSStream; + AVCodecParameters* pSCodecPar; + AVCodecContext* pSCodecCtx; + AVCodec* pSCodec; + int32_t sPTS; // msec - overall last subtitle PTS float fps; // frames per seconds int32_t bps_stream; // bits per seconds @@ -212,6 +216,7 @@ typedef struct { char acodec[64]; char vcodec[64]; + char scodec[64]; int32_t ready; -- cgit v1.2.3