diff options
author | Sven Gothel <[email protected]> | 2013-08-27 19:21:17 +0200 |
---|---|---|
committer | Sven Gothel <[email protected]> | 2013-08-27 19:21:17 +0200 |
commit | 554ec0576432194f050191bdf248a1462d542a6d (patch) | |
tree | c8f096f0cca0c07ae23d9d2cd81f91bd9334210f /src/jogl/classes/jogamp/opengl/util/av/impl | |
parent | 0de489019085246abb437454e9ac8fd6fc238692 (diff) |
GLMediaPlayer: Add camera input / FFMPEG: Fix 'av_packet' leak and add missing symbol 'av_realloc'.
- Add camera input
- Use URI w/ scheme 'camera' to determine camera input is desired,
use URI host as camera id.
E.g. 'camera://0' for 1st camera.
- AndroidGLMediaPlayerAPI14: Via 'Camera'
- FFMPEG*: Via libavdevice, device name and input format
- TODO: Add controls to manipulate camera if available
- FFMPEG*
- Add symbols
- avcodec_register_all
- av_realloc (was missing)
- avdevice_register_all
- Load libavdevice (opt)
- Camera:
- Use <ID> (windows) and /dev/video<ID> other OS
- simply find the input format in native code
- Support YUYV422 (used in video4linux2, etc.)
- Stuff 2x 16bpp (YUYV) into one RGBA pixel!
- Add texture format for 16bpp
- Add texture lookup shader
- Fix av_packet leak in readNextImpl(..)
- Restore orig pointer and size values,
we may have moved along within packet.
Then call av_free_packet().
- Use null AudioSink if audio-id is NONE
Diffstat (limited to 'src/jogl/classes/jogamp/opengl/util/av/impl')
5 files changed, 116 insertions, 56 deletions
diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java index ab16c5e5e..040d152f5 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGDynamicLibraryBundleInfo.java @@ -60,7 +60,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { private static final List<String> glueLibNames = new ArrayList<String>(); // none - private static final int symbolCount = 51; + private static final int symbolCount = 54; private static final String[] symbolNames = { "avcodec_version", "avformat_version", @@ -68,6 +68,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { /* 4 */ "avresample_version", // libavcodec + "avcodec_register_all", "avcodec_close", "avcodec_string", "avcodec_find_decoder", @@ -85,16 +86,17 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "av_free_packet", "avcodec_decode_audio4", // 53.25.0 (opt) "avcodec_decode_audio3", // 52.23.0 -/* 22 */ "avcodec_decode_video2", // 52.23.0 +/* 23 */ "avcodec_decode_video2", // 52.23.0 // libavutil "av_pix_fmt_descriptors", "av_frame_unref", // 55.0.0 (opt) + "av_realloc", "av_free", "av_get_bits_per_pixel", "av_samples_get_buffer_size", "av_get_bytes_per_sample", // 51.4.0 -/* 29 */ "av_opt_set_int", // 51.12.0 +/* 31 */ "av_opt_set_int", // 51.12.0 // libavformat "avformat_alloc_context", @@ -113,14 +115,17 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "avformat_network_init", // 53.13.0 (opt) "avformat_network_deinit", // 53.13.0 (opt) "avformat_find_stream_info", // 53.3.0 (opt) -/* 46 */ "av_find_stream_info", +/* 48 */ "av_find_stream_info", + // libavdevice +/* 49 */ "avdevice_register_all", // ??? + // libavresample "avresample_alloc_context", // 1.0.1 "avresample_open", "avresample_close", "avresample_free", -/* 51 */ "avresample_convert" +/* 54 */ "avresample_convert" }; // alternate symbol names @@ -139,7 +144,8 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { "avformat_seek_file", // ??? (opt) "avcodec_free_frame", // 54.28.0 (opt) "av_frame_unref", // 55.0.0 (opt) - + // libavdevice + "avdevice_register_all", // 53.0.0 (opt) // libavresample "avresample_version", // 1.0.1 "avresample_alloc_context", // 1.0.1 @@ -153,6 +159,7 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { private static final boolean ready; private static final boolean libsLoaded; private static final boolean avresampleLoaded; // optional + private static final boolean avdeviceLoaded; // optional static final VersionNumber avCodecVersion; static final VersionNumber avFormatVersion; static final VersionNumber avUtilVersion; @@ -164,14 +171,16 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { GLProfile.initSingleton(); boolean _ready = false; boolean[] _libsLoaded= { false }; + boolean[] _avdeviceLoaded= { false }; boolean[] _avresampleLoaded= { false }; VersionNumber[] _versions = new VersionNumber[4]; try { - _ready = initSymbols(_libsLoaded, _avresampleLoaded, _versions); + _ready = initSymbols(_libsLoaded, _avdeviceLoaded, _avresampleLoaded, _versions); } catch (Throwable t) { t.printStackTrace(); } libsLoaded = _libsLoaded[0]; + avdeviceLoaded = _avdeviceLoaded[0]; avresampleLoaded = _avresampleLoaded[0]; avCodecVersion = _versions[0]; avFormatVersion = _versions[1]; @@ -205,11 +214,13 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { } static boolean libsLoaded() { return libsLoaded; } + static boolean avDeviceLoaded() { return avdeviceLoaded; } static boolean avResampleLoaded() { return avresampleLoaded; } static FFMPEGNatives getNatives() { return natives; } static boolean initSingleton() { return ready; } - private static final boolean initSymbols(boolean[] libsLoaded, boolean[] avresampleLoaded, VersionNumber[] versions) { + private static final boolean initSymbols(boolean[] libsLoaded, boolean[] avdeviceLoaded, boolean[] avresampleLoaded, + VersionNumber[] versions) { libsLoaded[0] = false; final DynamicLibraryBundle dl = AccessController.doPrivileged(new PrivilegedAction<DynamicLibraryBundle>() { public DynamicLibraryBundle run() { @@ -221,7 +232,8 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { if(!avutilLoaded || !avformatLoaded || !avcodecLoaded) { throw new RuntimeException("FFMPEG Tool library incomplete: [ avutil "+avutilLoaded+", avformat "+avformatLoaded+", avcodec "+avcodecLoaded+"]"); } - avresampleLoaded[0] = dl.isToolLibLoaded(3); + avdeviceLoaded[0] = dl.isToolLibLoaded(3); + avresampleLoaded[0] = dl.isToolLibLoaded(4); libsLoaded[0] = true; if(symbolNames.length != symbolCount) { @@ -361,6 +373,16 @@ class FFMPEGDynamicLibraryBundleInfo implements DynamicLibraryBundleInfo { avcodec.add("avcodec-52"); // 0.7 libsList.add(avcodec); + final List<String> avdevice = new ArrayList<String>(); + avdevice.add("avdevice"); // default + + avdevice.add("libavdevice.so.54"); // dummy future proof + avdevice.add("libavdevice.so.53"); // 8 && 9 + + avdevice.add("avdevice-54"); // dummy future proof + avdevice.add("avdevice-53"); // 8 && 9 + libsList.add(avdevice); + final List<String> avresample = new ArrayList<String>(); avresample.add("avresample"); // default diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java index bcf4994b5..f46c5900c 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGMediaPlayer.java @@ -129,7 +129,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { System.err.println("LIB_AV Util : "+FFMPEGDynamicLibraryBundleInfo.avUtilVersion+" [cc "+avUtilMajorVersionCC+"]"); System.err.println("LIB_AV Format : "+FFMPEGDynamicLibraryBundleInfo.avFormatVersion+" [cc "+avFormatMajorVersionCC+"]"); System.err.println("LIB_AV Codec : "+FFMPEGDynamicLibraryBundleInfo.avCodecVersion+" [cc "+avCodecMajorVersionCC+"]"); - System.err.println("LIB_AV Resample: "+FFMPEGDynamicLibraryBundleInfo.avResampleVersion+" [cc "+avResampleMajorVersionCC+"]"); + System.err.println("LIB_AV Device : [loaded "+FFMPEGDynamicLibraryBundleInfo.avDeviceLoaded()+"]"); + System.err.println("LIB_AV Resample: "+FFMPEGDynamicLibraryBundleInfo.avResampleVersion+" [cc "+avResampleMajorVersionCC+", loaded "+FFMPEGDynamicLibraryBundleInfo.avResampleLoaded()+"]"); libAVVersionGood = avUtilMajorVersionCC == FFMPEGDynamicLibraryBundleInfo.avUtilVersion.getMajor() && avFormatMajorVersionCC == FFMPEGDynamicLibraryBundleInfo.avFormatVersion.getMajor() && avCodecMajorVersionCC == FFMPEGDynamicLibraryBundleInfo.avCodecVersion.getMajor() && @@ -160,6 +161,8 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { // Video // + private String texLookupFuncName = "ffmpegTexture2D"; + private boolean usesTexLookupShader = false; private PixelFormat vPixelFmt = null; private int vPlanes = 0; private int vBitsPerPixel = 0; @@ -205,8 +208,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { } } - public static final String dev_video = "/dev/video"; - private static final int dev_video_len = dev_video.length(); + public static final String dev_video_linux = "/dev/video"; @Override protected final void initStreamImpl(int vid, int aid) throws IOException { @@ -229,40 +231,34 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { System.err.println("initStream: p2 preferred "+preferredAudioFormat+", "+this); } - final int streamLocSLen = streamLocS.length(); - final String inFormat; + final boolean isCameraInput = null != cameraHostPart; final String resStreamLocS; - if( streamLocSLen == dev_video_len + 1 && streamLocS.startsWith(dev_video) ) { - final int index = Integer.valueOf( streamLocS.substring(streamLocSLen-1) ).intValue(); + if( isCameraInput ) { switch(Platform.OS_TYPE) { - case ANDROID: - // ?? - case FREEBSD: - case HPUX: - case LINUX: - case SUNOS: - resStreamLocS = streamLocS; - inFormat = "video4linux2"; - break; - case WINDOWS: - resStreamLocS = String.valueOf(index); - inFormat = "vfwcap"; - break; - case MACOS: - case OPENKODE: - default: - resStreamLocS = streamLocS; - inFormat = null; - break; + case ANDROID: + // ?? + case FREEBSD: + case HPUX: + case LINUX: + case SUNOS: + resStreamLocS = dev_video_linux + cameraHostPart; + break; + case WINDOWS: + resStreamLocS = cameraHostPart; + break; + case MACOS: + case OPENKODE: + default: + resStreamLocS = streamLocS; // FIXME: ?? + break; } } else { resStreamLocS = streamLocS; - inFormat = null; } final int aMaxChannelCount = audioSink.getMaxSupportedChannels(); final int aPrefSampleRate = preferredAudioFormat.sampleRate; // setStream(..) issues updateAttributes*(..), and defines avChosenAudioFormat, vid, aid, .. etc - natives.setStream0(moviePtr, resStreamLocS, inFormat, vid, aid, aMaxChannelCount, aPrefSampleRate); + natives.setStream0(moviePtr, resStreamLocS, isCameraInput, vid, aid, aMaxChannelCount, aPrefSampleRate); } @Override @@ -300,6 +296,10 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { System.err.println("initGL: p3 avChosen "+avChosenAudioFormat); } + if( STREAM_ID_NONE == aid ) { + audioSink.destroy(); + audioSink = AudioSinkFactory.createNull(); + } final boolean audioSinkOK = audioSink.init(avChosenAudioFormat, frameDuration, AudioSink.DefaultInitialQueueSize, AudioSink.DefaultQueueGrowAmount, audioQueueLimit); if( !audioSinkOK ) { System.err.println("AudioSink "+audioSink.getClass().getName()+" does not support "+avChosenAudioFormat+", using Null"); @@ -314,6 +314,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if( null != gl ) { int tf, tif=GL.GL_RGBA; // texture format and internal format + int tt = GL.GL_UNSIGNED_BYTE; switch(vBytesPerPixelPerPlane) { case 1: if( gl.isGL3ES3() ) { @@ -324,12 +325,20 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { tf = GL2ES2.GL_ALPHA; tif=GL2ES2.GL_ALPHA; singleTexComp = "a"; } break; - case 3: tf = GL2ES2.GL_RGB; tif=GL.GL_RGB; break; - case 4: tf = GL2ES2.GL_RGBA; tif=GL.GL_RGBA; break; + + case 2: if( vPixelFmt == PixelFormat.YUYV422 ) { + // YUYV422: // < packed YUV 4:2:2, 2x 16bpp, Y0 Cb Y1 Cr + // Stuffed into RGBA half width texture + tf = GL2ES2.GL_RGBA; tif=GL2ES2.GL_RGBA; break; + } else { + tf = GL2ES2.GL_RG; tif=GL2ES2.GL_RG; break; + } + case 3: tf = GL2ES2.GL_RGB; tif=GL.GL_RGB; break; + case 4: tf = GL2ES2.GL_RGBA; tif=GL.GL_RGBA; break; default: throw new RuntimeException("Unsupported bytes-per-pixel / plane "+vBytesPerPixelPerPlane); } setTextureFormat(tif, tf); - setTextureType(GL.GL_UNSIGNED_BYTE); + setTextureType(tt); } } @Override @@ -435,7 +444,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { */ void updateAttributes2(int pixFmt, int planes, int bitsPerPixel, int bytesPerPixelPerPlane, int lSz0, int lSz1, int lSz2, - int tWd0, int tWd1, int tWd2, int tH, + int tWd0, int tWd1, int tWd2, int vW, int vH, int audioSampleFmt, int audioSampleRate, int audioChannels, int audioSamplesPerFrameAndChannel) { vPixelFmt = PixelFormat.valueOf(pixFmt); @@ -446,23 +455,28 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { vTexWidth[0] = tWd0; vTexWidth[1] = tWd1; vTexWidth[2] = tWd2; switch(vPixelFmt) { - case YUV420P: + case YUV420P: // < planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples) + usesTexLookupShader = true; // YUV420P: Adding U+V on right side of fixed height texture, // since width is already aligned by decoder. // Y=w*h, Y=w/2*h/2, U=w/2*h/2 // w*h + 2 ( w/2 * h/2 ) // w*h + w*h/2 // 2*w/2 * h - texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = tH; + texWidth = vTexWidth[0] + vTexWidth[1]; texHeight = vH; + break; + case YUYV422: // < packed YUV 4:2:2, 2x 16bpp, Y0 Cb Y1 Cr - stuffed into RGBA half width texture + usesTexLookupShader = true; + texWidth = vTexWidth[0]; texHeight = vH; break; - // case PIX_FMT_YUYV422: case RGB24: case BGR24: case ARGB: case RGBA: case ABGR: case BGRA: - texWidth = vTexWidth[0]; texHeight = tH; + usesTexLookupShader = false; + texWidth = vTexWidth[0]; texHeight = vH; break; default: // FIXME: Add more formats ! throw new RuntimeException("Unsupported pixelformat: "+vPixelFmt); @@ -474,7 +488,7 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if(DEBUG) { System.err.println("audio: fmt "+aSampleFmt+", "+avChosenAudioFormat+", aFrameSize/fc "+audioSamplesPerFrameAndChannel); - System.err.println("video: fmt "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane); + System.err.println("video: fmt "+vW+"x"+vH+", "+vPixelFmt+", planes "+vPlanes+", bpp "+vBitsPerPixel+"/"+vBytesPerPixelPerPlane+", usesTexLookupShader "+usesTexLookupShader); for(int i=0; i<3; i++) { System.err.println("video: "+i+": "+vTexWidth[i]+"/"+vLinesize[i]); } @@ -494,15 +508,14 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if(State.Uninitialized == state) { throw new IllegalStateException("Instance not initialized: "+this); } - if(PixelFormat.YUV420P == vPixelFmt) { + if( usesTexLookupShader ) { if(null != desiredFuncName && desiredFuncName.length()>0) { - textureLookupFunctionName = desiredFuncName; + texLookupFuncName = desiredFuncName; } - return textureLookupFunctionName; + return texLookupFuncName; } return super.getTextureLookupFunctionName(desiredFuncName); } - private String textureLookupFunctionName = "ffmpegTexture2D"; /** * {@inheritDoc} @@ -515,11 +528,14 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { if(State.Uninitialized == state) { throw new IllegalStateException("Instance not initialized: "+this); } + if( !usesTexLookupShader ) { + return super.getTextureLookupFragmentShaderImpl(); + } final float tc_w_1 = (float)getWidth() / (float)texWidth; switch(vPixelFmt) { - case YUV420P: + case YUV420P: // < planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples) return - "vec4 "+textureLookupFunctionName+"(in "+getTextureSampler2DType()+" image, in vec2 texCoord) {\n"+ + "vec4 "+texLookupFuncName+"(in "+getTextureSampler2DType()+" image, in vec2 texCoord) {\n"+ " vec2 u_off = vec2("+tc_w_1+", 0.0);\n"+ " vec2 v_off = vec2("+tc_w_1+", 0.5);\n"+ " vec2 tc_half = texCoord*0.5;\n"+ @@ -536,8 +552,30 @@ public class FFMPEGMediaPlayer extends GLMediaPlayerImpl { " return vec4(r, g, b, 1);\n"+ "}\n" ; + case YUYV422: // < packed YUV 4:2:2, 2 x 16bpp, [Y0 Cb] [Y1 Cr] + // Stuffed into RGBA half width texture + return + "vec4 "+texLookupFuncName+"(in "+getTextureSampler2DType()+" image, in vec2 texCoord) {\n"+ + " "+ + " float y1,u,y2,v,y,r,g,b;\n"+ + " vec2 tc_halfw = vec2(texCoord.x*0.5, texCoord.y);\n"+ + " vec4 yuyv = texture2D(image, tc_halfw).rgba;\n"+ + " y1 = yuyv.r;\n"+ + " u = yuyv.g;\n"+ + " y2 = yuyv.b;\n"+ + " v = yuyv.a;\n"+ + " y = mix( y1, y2, mod(gl_FragCoord.x, 2) ); /* avoid branching! */\n"+ + " y = 1.1643*(y-0.0625);\n"+ + " u = u-0.5;\n"+ + " v = v-0.5;\n"+ + " r = y+1.5958*v;\n"+ + " g = y-0.39173*u-0.81290*v;\n"+ + " b = y+2.017*u;\n"+ + " return vec4(r, g, b, 1);\n"+ + "}\n" + ; default: // FIXME: Add more formats ! - return super.getTextureLookupFragmentShaderImpl(); + throw new InternalError("Add proper mapping of: vPixelFmt "+vPixelFmt+", usesTexLookupShader "+usesTexLookupShader); } } diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGNatives.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGNatives.java index 89cefbf17..9dd1ac74a 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGNatives.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGNatives.java @@ -59,7 +59,7 @@ interface FFMPEGNatives { * @param aPrefChannelCount * @param aPrefSampleRate */ - void setStream0(long moviePtr, String url, String inFormat, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); + void setStream0(long moviePtr, String url, boolean isCameraInput, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); void setGLFuncs0(long moviePtr, long procAddrGLTexSubImage2D, long procAddrGLGetError, long procAddrGLFlush, long procAddrGLFinish); int getVideoPTS0(long moviePtr); diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv08Natives.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv08Natives.java index 1c0f66e0c..3b2567655 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv08Natives.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv08Natives.java @@ -53,7 +53,7 @@ class FFMPEGv08Natives implements FFMPEGNatives { public native void destroyInstance0(long moviePtr); @Override - public native void setStream0(long moviePtr, String url, String inFormat, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); + public native void setStream0(long moviePtr, String url, boolean isCameraInput, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); @Override public native void setGLFuncs0(long moviePtr, long procAddrGLTexSubImage2D, long procAddrGLGetError, long procAddrGLFlush, long procAddrGLFinish); diff --git a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv09Natives.java b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv09Natives.java index 0bc26923c..6c56d3ccb 100644 --- a/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv09Natives.java +++ b/src/jogl/classes/jogamp/opengl/util/av/impl/FFMPEGv09Natives.java @@ -53,7 +53,7 @@ class FFMPEGv09Natives implements FFMPEGNatives { public native void destroyInstance0(long moviePtr); @Override - public native void setStream0(long moviePtr, String url, String inFormat, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); + public native void setStream0(long moviePtr, String url, boolean isCameraInput, int vid, int aid, int aMaxChannelCount, int aPrefSampleRate); @Override public native void setGLFuncs0(long moviePtr, long procAddrGLTexSubImage2D, long procAddrGLGetError, long procAddrGLFlush, long procAddrGLFinish); |