Bug 1472: Enhance AV Sync: Pass through PTS object in AudioSink, tracking last PTS value against System Clock Reference (SCR)

This allows passing through actual SCR with the PTS when enqueueing audio frames
and utilizing it when querying actual time lapsed PTS value (interpolation SCR to currentMillis).

4 files changed, 159 insertions, 19 deletions

diff --git a/src/java/com/jogamp/common/av/AudioSink.java b/src/java/com/jogamp/common/av/AudioSink.java
index 6f36e6f..becb3a6 100644
--- a/src/java/com/jogamp/common/av/AudioSink.java
+++ b/src/java/com/jogamp/common/av/AudioSink.java
@@ -39,8 +39,8 @@ public interface AudioSink {
 
     /** Initial audio queue size in milliseconds. {@value} ms, i.e. 16 {@link AudioFrame}s per 32 ms. See {@link #init(AudioFormat, float, int)}.*/
     public static final int DefaultQueueSize = 16 * 32; // 512 ms
-    /** Audio queue size w/ video in milliseconds. {@value} ms, i.e. 24 {@link AudioFrame}s per 32 ms. See {@link #init(AudioFormat, float, int)}.*/
-    public static final int DefaultQueueSizeWithVideo =  24 * 32; // 768 ms
+    /** Audio queue size w/ video in milliseconds. {@value} ms, i.e. 32 {@link AudioFrame}s per 32 ms. See {@link #init(AudioFormat, float, int)}.*/
+    public static final int DefaultQueueSizeWithVideo =  32 * 32; // 1024 ms
 
     /** Default {@link AudioFormat}, [type PCM, sampleRate 44100, sampleSize 16, channelCount 2, signed, fixedP, !planar, littleEndian]. */
     public static final AudioFormat DefaultFormat = new AudioFormat(44100, 16, 2, true /* signed */,
@@ -403,19 +403,14 @@ public interface AudioSink {
     public float getAvgFrameDuration();
 
     /**
-     * Return the current audio presentation timestamp (PTS) in milliseconds.
+     * Return the audio presentation timestamp ({@link PTS}).
      * <p>
      * In case implementation updates the audio buffer passively, consider using {@link #updateQueue()}.
      * </p>
-     * <p>
-     * The relative millisecond PTS since start of the presentation stored in integer
-     * covers a time span of 2'147'483'647 ms (see {@link Integer#MAX_VALUE}
-     * or 2'147'483 seconds or 24.855 days.
-     * </p>
      * @see #updateQueue()
      * @see #enqueueData(int, ByteBuffer, int)
      */
-    public int getPTS();
+    public PTS getPTS();
 
     /**
      * Return the last buffered audio presentation timestamp (PTS) in milliseconds.
@@ -453,9 +448,9 @@ public interface AudioSink {
      * Useful in case implementation only updates the buffer passively via {@link #enqueueData(int, ByteBuffer, int) enqueueing data}
      * to add new data to the queue and not on a event basis.
      * </p>
-     * @return the updated current audio PTS
+     * @return the updated {@link PTS} instance
      * @see #getPTS()
      * @see #enqueueData(int, ByteBuffer, int)
      */
-    public int updateQueue();
+    public PTS updateQueue();
 }
diff --git a/src/java/com/jogamp/common/av/PTS.java b/src/java/com/jogamp/common/av/PTS.java
new file mode 100644
index 0000000..891a323
--- /dev/null
+++ b/src/java/com/jogamp/common/av/PTS.java
@@ -0,0 +1,137 @@
+/**
+ * Copyright 2023 JogAmp Community. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are
+ * permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this list of
+ *       conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice, this list
+ *       of conditions and the following disclaimer in the documentation and/or other materials
+ *       provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY JogAmp Community ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JogAmp Community OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are those of the
+ * authors and should not be interpreted as representing official policies, either expressed
+ * or implied, of JogAmp Community.
+ */
+package com.jogamp.common.av;
+
+import com.jogamp.common.os.Clock;
+
+/**
+ * Presentation Timestamp (PTS) with added System Clock Reference (SCR) via
+ * {@link #set(long, int)} and its interpolation via {@link #get(long)}, as well as giving raw access via {@link #getLast()}.
+ * <p>
+ * The relative millisecond PTS since start of the presentation stored in integer
+ * covers a time span of 2'147'483'647 ms (see {@link Integer#MAX_VALUE}
+ * or 2'147'483 seconds or 24.855 days.
+ * </p>
+ */
+public final class PTS {
+    /** An external float value getter */
+    public static interface FloatValue {
+        float get();
+    }
+    private final FloatValue speed;
+    /** System Clock Reference (SCR) of last PTS update. */
+    private volatile long scr;
+    /** Last updated PTS value */
+    private volatile int pts;
+
+    /**
+     * Create new instance, initializing pts with {@link TimeFrameI#INVALID_PTS} and system-clock timestamp with zero.
+     * @param speed external {@link FloatValue} getter for playback speed.
+     * @see #set(long, int)
+     */
+    public PTS(final FloatValue speed) {
+        this.speed = speed;
+        this.scr = 0;
+        this.pts = TimeFrameI.INVALID_PTS;
+    }
+    /**
+     * Create new instance.
+     * @param speed external {@link FloatValue} getter for playback speed.
+     * @param scr System Clock Reference (SCR) in milliseconds of taken pts value, i.e. {@link Clock#currentMillis()}.
+     * @param pts the presentation timestamp (PTS) in milliseconds
+     * @see #set(long, int)
+     */
+    public PTS(final FloatValue speed, final long scr, final int pts) {
+        this.speed = speed;
+        set(scr, pts);
+    }
+
+    /** Returns true if {@link #getLast()} is unequal to {@link TimeFrameI#INVALID_PTS}. */
+    public boolean isValid() { return TimeFrameI.INVALID_PTS != pts; }
+
+    /** Returns true if {@link #getLast()} equals to {@link TimeFrameI#END_OF_STREAM_PTS}, indicating end of stream (EOS). */
+    public boolean isEOS() { return TimeFrameI.END_OF_STREAM_PTS == pts; }
+
+    /** Returns the System Clock Reference (SCR) in milliseconds of last PTS update via {@link #set(long, int)}. */
+    public long getSCR() { return scr; }
+    /** Returns the last updated PTS value via {@link #set(long, int)} w/o System Clock Reference (SCR) interpolation. */
+    public int getLast() { return pts; }
+    /** Returns the external playback speed. */
+    public float getSpeed() { return speed.get(); }
+
+    /**
+     * Updates the PTS value with given System Clock Reference (SCR) in milliseconds.
+     * @param scr System Clock Reference (SCR) in milliseconds of taken PTS value, i.e. {@link Clock#currentMillis()}.
+     * @param pts the presentation timestamp (PTS) in milliseconds
+     */
+    public void set(final long scr, final int pts) {
+        this.scr = scr;
+        this.pts = pts;
+    }
+    /** Sets the PTS value, see {@link #set(long, int)}. */
+    public void setPTS(final int pts) { this.pts = pts; }
+    /** Sets the System Clock Reference (SCR) in milliseconds of last PTS update, see {@link #set(long, int)}. */
+    public void setSCR(final long currentMillis) { scr = currentMillis; }
+
+    /**
+     * Updates the PTS value with values from other {@link PTS} instance.
+     * @param other source {@link PTS} values
+     * @see #get(long)
+     */
+    public void set(final PTS other) {
+        this.scr = other.getSCR();
+        this.pts = other.getLast();
+    }
+
+    /**
+     * Returns the {@link #getLast() last updated PTS}, interpolated by {@link #getSCR() System Clock Reference (SCR)} delta to given {@code currentMillis} and playback {@link #getSpeed() speed}.
+     * <pre>
+     *      last_pts + ( currentMillis - SCR ) * speed
+     * </pre>
+     * @param currentMillis current system clock in milliseconds, i.e. {@link Clock#currentMillis()}.
+     * @see #set(long, int)
+     */
+    public int get(final long currentMillis) {
+        return pts + (int) ( ( currentMillis - scr ) * speed.get() );
+    }
+
+    /** Returns {@link #getLast()} - rhs.{@link #getLast()}. */
+    public int diffLast(final PTS rhs) {
+        return this.pts - rhs.getLast();
+    }
+
+    /** Returns {@link #get(long)} - rhs.{@link #get(long)}. */
+    public int diff(final long currentMillis, final PTS rhs) {
+        return get(currentMillis) - rhs.get(currentMillis);
+    }
+
+    @Override
+    public String toString() { return String.valueOf(pts); }
+
+    public String toString(final long currentMillis) { return "last "+pts+" ms, current "+get(currentMillis)+" ms"; }
+}
diff --git a/src/java/jogamp/common/av/JavaSoundAudioSink.java b/src/java/jogamp/common/av/JavaSoundAudioSink.java
index 556ada4..c556e92 100644
--- a/src/java/jogamp/common/av/JavaSoundAudioSink.java
+++ b/src/java/jogamp/common/av/JavaSoundAudioSink.java
@@ -36,6 +36,9 @@ import javax.sound.sampled.SourceDataLine;
 
 import com.jogamp.common.av.AudioFormat;
 import com.jogamp.common.av.AudioSink;
+import com.jogamp.common.av.PTS;
+import com.jogamp.common.av.TimeFrameI;
+import com.jogamp.common.os.Clock;
 
 /***
  * JavaSound Audio Sink
@@ -65,8 +68,8 @@ public final class JavaSoundAudioSink implements AudioSink {
     private int userMaxChannels = 8;
     private AudioFormat preferredFormat = null;
     private AudioFormat chosenFormat = null;
-
     private volatile boolean playRequested = false;
+    private final PTS pts = new PTS( () -> { return playRequested ? 1f : 0f; } );
     private float volume = 1.0f;
 
     static {
@@ -243,6 +246,7 @@ public final class JavaSoundAudioSink implements AudioSink {
     public void flush() {
         if( null != auline ) {
             playRequested = false;
+            pts.set(0, TimeFrameI.INVALID_PTS);
             auline.stop();
             auline.flush();
         }
@@ -290,6 +294,7 @@ public final class JavaSoundAudioSink implements AudioSink {
             bytesLeft -= len;
             written += len;
         }
+        this.pts.set(Clock.currentMillis(), pts);
         playImpl();
         return new AudioDataFrame(pts, Math.round(1000f*chosenFormat.getBytesDuration(byteCount)), byteBuffer, byteCount);
     }
@@ -315,10 +320,10 @@ public final class JavaSoundAudioSink implements AudioSink {
     }
 
     @Override
-    public final int getPTS() { return 0; } // FIXME
+    public final PTS getPTS() { return pts; }
 
     @Override
-    public final int updateQueue() { return 0; } // FIXME
+    public final PTS updateQueue() { return pts; } // FIXME
 
     @Override
     public int getLastBufferedPTS() { return 0; } // FIXME
diff --git a/src/java/jogamp/common/av/NullAudioSink.java b/src/java/jogamp/common/av/NullAudioSink.java
index d38c040..abc610f 100644
--- a/src/java/jogamp/common/av/NullAudioSink.java
+++ b/src/java/jogamp/common/av/NullAudioSink.java
@@ -31,12 +31,15 @@ import java.nio.ByteBuffer;
 
 import com.jogamp.common.av.AudioFormat;
 import com.jogamp.common.av.AudioSink;
+import com.jogamp.common.av.PTS;
+import com.jogamp.common.av.TimeFrameI;
+import com.jogamp.common.os.Clock;
 
 public final class NullAudioSink implements AudioSink {
 
     private volatile float playSpeed = 1.0f;
     private volatile boolean playRequested = false;
-    private volatile int playingPTS = AudioFrame.INVALID_PTS;
+    private final PTS pts = new PTS( () -> { return playRequested ? 1f : 0f; } );
     private float volume = 1.0f;
 
     private AudioFormat chosenFormat;
@@ -153,7 +156,7 @@ public final class NullAudioSink implements AudioSink {
     }
 
     @Override
-    public void flush() { }
+    public void flush() { pts.set(0, TimeFrameI.INVALID_PTS); }
 
     @Override
     public void destroy() {
@@ -180,10 +183,10 @@ public final class NullAudioSink implements AudioSink {
     public float getAvgFrameDuration() { return 0f; }
 
     @Override
-    public final int getPTS() { return playingPTS; }
+    public final PTS getPTS() { return pts; }
 
     @Override
-    public final int updateQueue() { return playingPTS; }
+    public final PTS updateQueue() { return pts; }
 
     @Override
     public int getLastBufferedPTS() { return 0; }
@@ -196,7 +199,7 @@ public final class NullAudioSink implements AudioSink {
         if( !available || null == chosenFormat ) {
             return null;
         }
-        playingPTS = pts;
+        this.pts.set(Clock.currentMillis(), pts);
         return null;
     }
 }