GraphUI MediaButton/MediaPlayer-Widget: Add working subtitle (text + ass/saa) support via GLMediaPlayer/FFMPEGMediaPlayer

2 files changed, 174 insertions, 6 deletions

diff --git a/src/graphui/classes/com/jogamp/graph/ui/shapes/MediaButton.java b/src/graphui/classes/com/jogamp/graph/ui/shapes/MediaButton.java
index 543248746..931f12341 100644
--- a/src/graphui/classes/com/jogamp/graph/ui/shapes/MediaButton.java
+++ b/src/graphui/classes/com/jogamp/graph/ui/shapes/MediaButton.java
@@ -28,11 +28,24 @@
 package com.jogamp.graph.ui.shapes;
 
 import com.jogamp.opengl.GL2ES2;
+import com.jogamp.opengl.GLProfile;
+
+import java.util.ArrayList;
+import java.util.List;
+
 import com.jogamp.common.av.AudioSink;
+import com.jogamp.common.os.Clock;
 import com.jogamp.common.util.InterruptSource;
 import com.jogamp.graph.curve.Region;
 import com.jogamp.graph.curve.opengl.RegionRenderer;
+import com.jogamp.graph.font.Font;
 import com.jogamp.graph.ui.GraphShape;
+import com.jogamp.graph.ui.Scene;
+import com.jogamp.math.Vec4f;
+import com.jogamp.math.geom.AABBox;
+import com.jogamp.math.util.PMVMatrix4f;
+import com.jogamp.opengl.util.av.ASSEventLine;
+import com.jogamp.opengl.util.av.ASSEventListener;
 import com.jogamp.opengl.util.av.GLMediaPlayer;
 import com.jogamp.opengl.util.av.GLMediaPlayer.GLMediaEventListener;
 import com.jogamp.opengl.util.av.GLMediaPlayer.StreamException;
@@ -56,25 +69,80 @@ import com.jogamp.opengl.util.texture.TextureSequence.TextureFrame;
  * </p>
  */
 public class MediaButton extends TexSeqButton {
+    private final boolean DEBUG = false;
     private boolean verbose = false;
 
+    private final Label subLabel;
+    private final float subZOffset;
+    private boolean subEnabled;
+    private float subLineHeightPct;
+    private final List<ASSEventLine> assEventQueue = new ArrayList<ASSEventLine>();
+    private final Object assEventLock = new Object();
+
+    public MediaButton(final int renderModes, final float width, final float height, final GLMediaPlayer mPlayer) {
+        this(renderModes, width, height, mPlayer, null, 0);
+    }
+
     /**
+     *
      * @param renderModes
      * @param width
      * @param height
      * @param mPlayer
-     * @param mPlayerListener
+     * @param subFont subtitle font
+     * @param subLineHeightPct one subtitle line height percentage of this shape, default is 0.1f
      */
-    public MediaButton(final int renderModes, final float width,
-                       final float height, final GLMediaPlayer mPlayer) {
+    public MediaButton(final int renderModes, final float width, final float height, final GLMediaPlayer mPlayer,
+                       final Font subFont, final float subLineHeightPct)
+    {
         super(renderModes & ~Region.AA_RENDERING_MASK, width, height, mPlayer);
 
         setColor(1.0f, 1.0f, 1.0f, 0.0f);
         setPressedColorMod(0.9f, 0.9f, 0.9f, 0.7f);
         setToggleOffColorMod(0.8f, 0.8f, 0.8f, 1.0f);
         setToggleOnColorMod(1.0f, 1.0f, 1.0f, 1.0f);
+
+        mPlayer.setASSEventListener(assEventListener);
+
+        final Font f;
+        if( null != subFont ) {
+            f = subFont;
+            subEnabled = true;
+        } else {
+            f = Scene.getDefaultFont();
+            subEnabled = false;
+        }
+        this.subZOffset = Button.DEFAULT_LABEL_ZOFFSET;
+        this.subLineHeightPct = subLineHeightPct;
+        this.subLabel = new Label(renderModes, f, "");
+        this.subLabel.setColor( new Vec4f( 1f, 1, 0f, 1.0f ) );
+        this.subLabel.moveTo(0, 0, subZOffset);
+    }
+
+    /**
+     * Sets subtitle parameter
+     * @param subFont subtitle font
+     * @param subLineHeightPct one subtitle line height percentage of this shape, default is 0.1f
+     */
+    public void setSubtitleParams(final Font subFont, final float subLineHeightPct) {
+        this.subLabel.setFont(subFont);
+        this.subLineHeightPct = subLineHeightPct;
+        this.subEnabled = true;
     }
 
+    public final ASSEventListener getASSEventListener() { return assEventListener; }
+    private final ASSEventListener assEventListener = new ASSEventListener() {
+        @Override
+        public void run(final ASSEventLine e) {
+            synchronized( assEventLock ) {
+                assEventQueue.add(e);
+                if( DEBUG ) {
+                    System.err.println("MediaButton: GOT #"+assEventQueue.size()+": "+e);
+                }
+            }
+        }
+    };
+
     public MediaButton setVerbose(final boolean v) { verbose = v; return this; }
 
     /**
@@ -137,7 +205,12 @@ public class MediaButton extends TexSeqButton {
     volatile boolean resetGL = true;
 
     @Override
-    public void draw(final GL2ES2 gl, final RegionRenderer renderer) {
+    protected void addShapeToRegion(final GLProfile glp, final GL2ES2 gl) {
+        super.addShapeToRegion(glp, gl);
+    }
+
+    @Override
+    protected final void drawImpl0(final GL2ES2 gl, final RegionRenderer renderer, final Vec4f rgba) {
         final GLMediaPlayer mPlayer = (GLMediaPlayer)texSeq;
         if( resetGL ) {
             resetGL = false;
@@ -150,8 +223,83 @@ public class MediaButton extends TexSeqButton {
                 e.printStackTrace();
             }
         }
-        super.draw(gl, renderer);
+        super.drawImpl0(gl, renderer, rgba);
+        if( subEnabled ) {
+            drawSubtitle(gl, renderer);
+        }
         markStateDirty(); // keep on going
     };
+    protected final void drawSubtitle(final GL2ES2 gl, final RegionRenderer renderer) {
+        // dequeue and earmark new subtitle
+        final ASSEventLine ass;
+        final boolean newASS;
+        {
+            final ASSEventLine gotASS;
+            synchronized( assEventLock ) {
+                if( assEventQueue.size() > 0 ) {
+                    gotASS = assEventQueue.remove(0);
+                } else {
+                    gotASS = null;
+                }
+            }
+            final ASSEventLine lastASS = draw_lastASS;
+            if( null == gotASS || gotASS == lastASS ) {
+                ass = lastASS;
+                newASS = false;
+            } else {
+                draw_lastASS = gotASS;
+                ass = gotASS;
+                newASS = true;
+            }
+        }
+        // drop or draw (update label for new subtitle)
+        final GLMediaPlayer mPlayer = (GLMediaPlayer)texSeq;
+        final int pts;
+        {
+            // Well .. which one? So pick the lowest PTS to be more tolerant.
+            final int ptsS = mPlayer.getPTS().get(Clock.currentMillis());
+            final int ptsV = mPlayer.getVideoPTS();
+            pts = Math.min(ptsS, ptsV);
+        }
+        final boolean drawASS;
+        if( null == ass ) {
+            draw_lastASS = null;
+            drawASS = false;
+        } else if( ass.pts_end < pts && ass.getDuration() > 1000 ) { // min duration 1s, broken ASS have <= 3ms
+            if( DEBUG ) {
+                System.err.println("MediaButton: Drop: pts "+pts+", "+ass);
+            }
+            draw_lastASS = null;
+            drawASS = false;
+        } else {
+            drawASS = true;
+            if( newASS ) {
+                subLabel.setText(ass.text);
+                final AABBox subBox = subLabel.getBounds(gl.getGLProfile());
+                final float subLineHeight = subBox.getHeight() / ass.lines;
+                final float maxWidth = this.box.getWidth() * 0.95f;
+                float scale = ( this.box.getHeight() * subLineHeightPct ) / subLineHeight;
+                if( scale * subBox.getWidth() > maxWidth ) {
+                    scale = maxWidth / subBox.getWidth();
+                }
+                subLabel.setScale(scale, scale, 1);
+                final float dx = ( this.box.getWidth() - maxWidth ) * 0.5f;
+                final float dy = subLineHeight * scale * 0.25f;
+                this.subLabel.moveTo(dx, dy, subZOffset);
+                if( DEBUG ) {
+                    System.err.println("MediaButton: NEXT pts "+pts+", "+ass);
+                }
+            }
+        }
+        if( drawASS ) {
+            final PMVMatrix4f pmv = renderer.getMatrix();
+            pmv.pushMv();
+            subLabel.applyMatToMv(pmv);
+            subLabel.draw(gl, renderer);
+            pmv.popMv();
+        }
+
+    }
+    private ASSEventLine draw_lastASS;
 
 }
diff --git a/src/graphui/classes/com/jogamp/graph/ui/widgets/MediaPlayer.java b/src/graphui/classes/com/jogamp/graph/ui/widgets/MediaPlayer.java
index dae0700f8..4cc53c98e 100644
--- a/src/graphui/classes/com/jogamp/graph/ui/widgets/MediaPlayer.java
+++ b/src/graphui/classes/com/jogamp/graph/ui/widgets/MediaPlayer.java
@@ -54,6 +54,7 @@ import com.jogamp.graph.ui.shapes.Label;
 import com.jogamp.graph.ui.shapes.MediaButton;
 import com.jogamp.graph.ui.shapes.Rectangle;
 import com.jogamp.graph.ui.widgets.RangeSlider.SliderAdapter;
+import com.jogamp.math.FloatUtil;
 import com.jogamp.math.Vec2f;
 import com.jogamp.math.Vec3f;
 import com.jogamp.math.Vec4f;
@@ -89,6 +90,8 @@ public class MediaPlayer extends Widget {
     private static final float AlphaBlend = 0.3f;
     private static final float KnobScale = 3f;
 
+    private final MediaButton mButton;
+
     /**
      * Constructs a {@link MediaPlayer}, i.e. its shapes and controls.
      * @param renderModes Graph's {@link Region} render modes, see {@link GLRegion#create(GLProfile, int, TextureSequence) create(..)}.
@@ -108,6 +111,7 @@ public class MediaPlayer extends Widget {
 
         final Font fontInfo = Scene.getDefaultFont(), fontSymbols = Scene.getSymbolsFont();
         if( null == fontInfo || null == fontSymbols ) {
+            mButton = null;
             return;
         }
         final float zEpsilon = scene.getZEpsilon(16);
@@ -129,7 +133,7 @@ public class MediaPlayer extends Widget {
         this.setBorderColor(BorderColor).setBorder(BorderSz);
         this.setInteractive(true).setFixedARatioResize(true);
 
-        final MediaButton mButton = new MediaButton(renderModes, aratio, 1, mPlayer);
+        mButton = new MediaButton(renderModes, aratio, 1, mPlayer, fontInfo, 0.1f);
         mButton.setName("mp.mButton").setInteractive(false);
         mButton.setPerp().setPressedColorMod(1f, 1f, 1f, 0.85f);
 
@@ -484,8 +488,13 @@ public class MediaPlayer extends Widget {
                         fontSymbols.getUTF16String("zoom_out_map"), fontSymbols.getUTF16String("zoom_in_map"),  CtrlButtonWidth, CtrlButtonHeight, zEpsilon);
                 button.setName("zoom");
                 button.setSpacing(SymSpacing, FixedSymSize).setPerp().setColor(CtrlCellCol);
+                final boolean toggleBorder = FloatUtil.isEqual(1f, zoomSize);
                 button.onToggle( (final Shape s) -> {
                     if( s.isToggleOn() ) {
+                        if( toggleBorder ) {
+                            MediaPlayer.this.setBorder(0f);
+                            System.err.println("ZOOM: border off");
+                        }
                         final AABBox sbox = scene.getBounds();
                         final Group parent = this.getParent();
                         final float sx = sbox.getWidth() * zoomSize / this.getScaledWidth();
@@ -516,6 +525,9 @@ public class MediaPlayer extends Widget {
                         // System.err.println("Zoom: R "+_zoomReplacement);
                         zoomReplacement.set( _zoomReplacement );
                     } else {
+                        if( toggleBorder ) {
+                            MediaPlayer.this.setBorder(BorderSz);
+                        }
                         final Vec3f _zoomOrigScale = zoomOrigScale.getAndSet(null);
                         final Vec3f _zoomOrigPos = zoomOrigPos.getAndSet(null);
                         final Shape _zoomReplacement = zoomReplacement.getAndSet(null);
@@ -594,6 +606,14 @@ public class MediaPlayer extends Widget {
         ctrlSlider.getKnob().setDraggable(true);
     }
 
+    /**
+     * Sets subtitle parameter
+     * @param subFont subtitle font
+     * @param subLineHeightPct one subtitle line height percentage of this shape, default is 0.1f
+     */
+    public void setSubtitleParams(final Font subFont, final float subLineHeightPct) {
+        if( null != mButton ) {
+            mButton.setSubtitleParams(subFont, subLineHeightPct);
         }
     }