diff options
author | Sven Gothel <[email protected]> | 2014-09-29 03:57:30 +0200 |
---|---|---|
committer | Sven Gothel <[email protected]> | 2014-09-29 03:57:30 +0200 |
commit | 00a9ee70054872712017b5a14b19aa92068c8420 (patch) | |
tree | 755dab002246d461f422218b75570cf7d6ef2042 | |
parent | 92a6d2c1476fd562721f231f89afba9342ed8a20 (diff) |
Bug 1080 - Refine MappedByteBuffer*Stream impl. and API [doc], adding stream to stream copy as well as direct memory mapped ByteBuffer access
7 files changed, 514 insertions, 93 deletions
diff --git a/make/scripts/runtest.sh b/make/scripts/runtest.sh index c2f3764..778fa62 100755 --- a/make/scripts/runtest.sh +++ b/make/scripts/runtest.sh @@ -124,7 +124,8 @@ function onetest() { #onetest com.jogamp.common.nio.TestPointerBufferEndian 2>&1 | tee -a $LOG #onetest com.jogamp.common.nio.TestStructAccessorEndian 2>&1 | tee -a $LOG #onetest com.jogamp.common.nio.TestByteBufferInputStream 2>&1 | tee -a $LOG -onetest com.jogamp.common.nio.TestByteBufferOutputStream 2>&1 | tee -a $LOG +#onetest com.jogamp.common.nio.TestByteBufferOutputStream 2>&1 | tee -a $LOG +onetest com.jogamp.common.nio.TestByteBufferCopyStream 2>&1 | tee -a $LOG #onetest com.jogamp.common.os.TestElfReader01 2>&1 | tee -a $LOG #onetest com.jogamp.gluegen.PCPPTest 2>&1 | tee -a $LOG #onetest com.jogamp.gluegen.test.junit.generation.Test1p1JavaEmitter 2>&1 | tee -a $LOG diff --git a/src/java/com/jogamp/common/nio/ByteBufferInputStream.java b/src/java/com/jogamp/common/nio/ByteBufferInputStream.java index 5b6f121..b74360f 100644 --- a/src/java/com/jogamp/common/nio/ByteBufferInputStream.java +++ b/src/java/com/jogamp/common/nio/ByteBufferInputStream.java @@ -126,10 +126,14 @@ public class ByteBufferInputStream extends InputStream { public final int read(final byte[] b, final int off, final int len) { if (b == null) { throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { + } else if( off < 0 || + len < 0 || + off > b.length || + off + len > b.length || + off + len < 0 + ) { throw new IndexOutOfBoundsException("offset "+off+", length "+len+", b.length "+b.length); - } - if ( 0 == len ) { + } else if ( 0 == len ) { return 0; } final int totalRem = buf.remaining(); @@ -152,8 +156,7 @@ public class ByteBufferInputStream extends InputStream { throw new NullPointerException(); } else if (len < 0 || len > b.remaining()) { throw new IndexOutOfBoundsException("length "+len+", b "+b); - } - if ( 0 == len ) { + } else if ( 0 == len ) { return 0; } final int remaining = buf.remaining(); diff --git a/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java b/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java index 67cbbbe..5f91f64 100644 --- a/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java +++ b/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java @@ -34,7 +34,6 @@ import java.io.RandomAccessFile; import java.lang.ref.WeakReference; import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.security.AccessController; @@ -45,11 +44,15 @@ import jogamp.common.Debug; import com.jogamp.common.os.Platform; /** - * An {@link InputStream} implementation based on an underlying {@link MappedByteBuffer} - * supporting {@link #markSupported() mark}. + * An {@link InputStream} implementation based on an underlying {@link FileChannel}'s memory mapped {@link ByteBuffer}, + * {@link #markSupported() supporting} {@link #mark(int) mark} and {@link #reset()}. * <p> - * Intended to be utilized with a {@link MappedByteBuffer memory-mapped} {@link FileChannel#map(MapMode, long, long) FileChannel} - * beyond its size limitation of {@link Integer#MAX_VALUE}.<br> + * Implementation allows full memory mapped {@link ByteBuffer} coverage via {@link FileChannel#map(MapMode, long, long) FileChannel} + * beyond its size limitation of {@link Integer#MAX_VALUE} utilizing an array of {@link ByteBuffer} slices.<br> + * </p> + * <p> + * Implementation further allows full random access via {@link #position()} and {@link #position(long)} + * and accessing the memory mapped {@link ByteBuffer} slices directly via {@link #currentSlice()} and {@link #nextSlice()}. * </p> * @since 2.3.0 */ @@ -150,7 +153,7 @@ public class MappedByteBufferInputStream extends InputStream { private FileResizeOp fileResizeOp = NoFileResize; private int sliceCount; - ByteBuffer[] slices; + private ByteBuffer[] slices; private WeakReference<ByteBuffer>[] slices2GC; private long totalSize; @@ -162,10 +165,10 @@ public class MappedByteBufferInputStream extends InputStream { private boolean hasCleaner; private CacheMode cmode; - int currSlice; + private int sliceIdx; private long mark; - public final void dbgDump(final String prefix, final PrintStream out) { + final void dbgDump(final String prefix, final PrintStream out) { long fcSz = 0, pos = 0, rem = 0; try { fcSz = fc.size(); @@ -184,12 +187,12 @@ public class MappedByteBufferInputStream extends InputStream { out.println(prefix+" refCount "+refCount+", fcSize "+fcSz+", totalSize "+totalSize); out.println(prefix+" position "+pos+", remaining "+rem); out.println(prefix+" mmode "+mmode+", cmode "+cmode+", fileResizeOp "+fileResizeOp); - out.println(prefix+" slice "+currSlice+" / "+sliceCount+" ("+sliceCount2+")"); + out.println(prefix+" slice "+sliceIdx+" / "+sliceCount+" ("+sliceCount2+")"); out.println(prefix+" sliceShift "+sliceShift+" -> "+(1L << sliceShift)); } MappedByteBufferInputStream(final FileChannel fc, final FileChannel.MapMode mmode, final CacheMode cmode, - final int sliceShift, final long totalSize, final int currSlice) throws IOException { + final int sliceShift, final long totalSize, final int currSliceIdx) throws IOException { this.sliceShift = sliceShift; this.fc = fc; this.mmode = mmode; @@ -207,16 +210,16 @@ public class MappedByteBufferInputStream extends InputStream { this.hasCleaner = false; this.cmode = cmode; - this.currSlice = currSlice; + this.sliceIdx = currSliceIdx; this.mark = -1; - slice(currSlice).position(0); + currentSlice().position(0); } /** * Creates a new instance using the given {@link FileChannel}. * <p> - * The {@link MappedByteBuffer} slices will be mapped lazily at first usage. + * The {@link ByteBuffer} slices will be mapped lazily at first usage. * </p> * @param fileChannel the file channel to be mapped lazily. * @param mmode the map mode, default is {@link FileChannel.MapMode#READ_ONLY}. @@ -235,7 +238,7 @@ public class MappedByteBufferInputStream extends InputStream { * Creates a new instance using the given {@link FileChannel}, * given mapping-mode, given cache-mode and the {@link #DEFAULT_SLICE_SHIFT}. * <p> - * The {@link MappedByteBuffer} slices will be mapped lazily at first usage. + * The {@link ByteBuffer} slices will be mapped lazily at first usage. * </p> * @param fileChannel the file channel to be used. * @param mmode the map mode, default is {@link FileChannel.MapMode#READ_ONLY}. @@ -251,7 +254,7 @@ public class MappedByteBufferInputStream extends InputStream { * {@link FileChannel.MapMode#READ_ONLY read-only} mapping mode, {@link CacheMode#FLUSH_PRE_SOFT} * and the {@link #DEFAULT_SLICE_SHIFT}. * <p> - * The {@link MappedByteBuffer} slices will be mapped {@link FileChannel.MapMode#READ_ONLY} lazily at first usage. + * The {@link ByteBuffer} slices will be mapped {@link FileChannel.MapMode#READ_ONLY} lazily at first usage. * </p> * @param fileChannel the file channel to be used. * @throws IOException @@ -279,12 +282,14 @@ public class MappedByteBufferInputStream extends InputStream { } fc.close(); mark = -1; - currSlice = -1; + sliceIdx = -1; super.close(); } } } + final FileChannel.MapMode getMapMode() { return mmode; } + /** * @param fileResizeOp the new {@link FileResizeOp}. * @throws IllegalStateException if attempting to set the {@link FileResizeOp} to a different value than before @@ -345,7 +350,7 @@ public class MappedByteBufferInputStream extends InputStream { sliceCount = 0; totalSize = 0; mark = -1; - currSlice = 0; + sliceIdx = 0; } else { final long prePosition = position(); @@ -353,7 +358,7 @@ public class MappedByteBufferInputStream extends InputStream { final int newSliceCount = (int)( ( newTotalSize + ( sliceSize - 1 ) ) / sliceSize ); @SuppressWarnings("unchecked") final WeakReference<ByteBuffer>[] newSlices2GC = new WeakReference[ newSliceCount ]; - final MappedByteBuffer[] newSlices = new MappedByteBuffer[ newSliceCount ]; + final ByteBuffer[] newSlices = new ByteBuffer[ newSliceCount ]; final int copySliceCount = Math.min(newSliceCount, sliceCount-1); // drop last (resize) if( 0 < copySliceCount ) { System.arraycopy(slices2GC, 0, newSlices2GC, 0, copySliceCount); @@ -399,46 +404,64 @@ public class MappedByteBufferInputStream extends InputStream { public final synchronized MappedByteBufferOutputStream getOutputStream(final FileResizeOp fileResizeOp) throws IllegalStateException, IOException { - if( FileChannel.MapMode.READ_ONLY == mmode ) { - throw new IOException("FileChannel map-mode is read-only"); - } checkOpen(); - setFileResizeOp(fileResizeOp); + final MappedByteBufferOutputStream res = new MappedByteBufferOutputStream(this, fileResizeOp); refCount++; - this.fileResizeOp = null != fileResizeOp ? fileResizeOp : NoFileResize; - return new MappedByteBufferOutputStream(this); + return res; } - final synchronized ByteBuffer slice(final int i) throws IOException { - if ( null != slices[i] ) { - return slices[i]; + /** + * Return the mapped {@link ByteBuffer} slice at the current {@link #position()}. + * <p> + * Due to the nature of using sliced buffers mapping the whole region, + * user has to determine whether the returned buffer covers the desired region + * and may fetch the {@link #nextSlice()} until satisfied.<br> + * It is also possible to repeat this operation after reposition the stream via {@link #position(long)} + * or {@link #skip(long)} to a position within the next block, similar to {@link #nextSlice()}. + * </p> + * @throws IOException if a buffer slice operation failed. + */ + public final synchronized ByteBuffer currentSlice() throws IOException { + if ( null != slices[sliceIdx] ) { + return slices[sliceIdx]; } else { if( CacheMode.FLUSH_PRE_SOFT == cmode ) { - final WeakReference<ByteBuffer> ref = slices2GC[i]; + final WeakReference<ByteBuffer> ref = slices2GC[sliceIdx]; if( null != ref ) { final ByteBuffer mbb = ref.get(); - slices2GC[i] = null; + slices2GC[sliceIdx] = null; if( null != mbb ) { - slices[i] = mbb; + slices[sliceIdx] = mbb; return mbb; } } } - final long pos = (long)i << sliceShift; - slices[i] = fc.map(mmode, pos, Math.min(1L << sliceShift, totalSize - pos)); - return slices[i]; + final long pos = (long)sliceIdx << sliceShift; + slices[sliceIdx] = fc.map(mmode, pos, Math.min(1L << sliceShift, totalSize - pos)); + return slices[sliceIdx]; } } - final synchronized boolean nextSlice() throws IOException { - if ( currSlice < sliceCount - 1 ) { + + /** + * Return the <i>next</i> mapped {@link ByteBuffer} slice from the current {@link #position()}, + * implicitly setting {@link #position(long)} to the start of the returned <i>next</i> slice, + * see {@link #currentSlice()}. + * <p> + * If no subsequent slice is available, {@code null} is being returned. + * </p> + * @throws IOException if a buffer slice operation failed. + */ + public final synchronized ByteBuffer nextSlice() throws IOException { + if ( sliceIdx < sliceCount - 1 ) { if( CacheMode.FLUSH_NONE != cmode ) { - flushSlice(currSlice); + flushSlice(sliceIdx); } - currSlice++; - slice( currSlice ).position( 0 ); - return true; + sliceIdx++; + final ByteBuffer slice = currentSlice(); + slice.position( 0 ); + return slice; } else { - return false; + return null; } } @@ -573,7 +596,7 @@ public class MappedByteBufferInputStream extends InputStream { // @Override public final synchronized long position() throws IOException { if( 0 < refCount ) { - return ( (long)currSlice << sliceShift ) + slice( currSlice ).position(); + return ( (long)sliceIdx << sliceShift ) + currentSlice().position(); } else { return 0; } @@ -594,9 +617,9 @@ public class MappedByteBufferInputStream extends InputStream { if ( totalSize < newPosition || 0 > newPosition ) { throw new IllegalArgumentException("new position "+newPosition+" not within [0.."+totalSize+"]"); } - final int preSlice = currSlice; + final int preSlice = sliceIdx; positionImpl( newPosition ); - if( CacheMode.FLUSH_NONE != cmode && preSlice != currSlice) { + if( CacheMode.FLUSH_NONE != cmode && preSlice != sliceIdx) { flushSlice(preSlice); } return this; @@ -604,12 +627,12 @@ public class MappedByteBufferInputStream extends InputStream { private final synchronized void positionImpl( final long newPosition ) throws IOException { if ( totalSize == newPosition ) { // EOF, pos == maxPos + 1 - currSlice = Math.max(0, sliceCount - 1); // handle zero size - final ByteBuffer s = slice( currSlice ); + sliceIdx = Math.max(0, sliceCount - 1); // handle zero size + final ByteBuffer s = currentSlice(); s.position( s.capacity() ); } else { - currSlice = (int)( newPosition >>> sliceShift ); - slice( currSlice ).position( (int)( newPosition - ( (long)currSlice << sliceShift ) ) ); + sliceIdx = (int)( newPosition >>> sliceShift ); + currentSlice().position( (int)( newPosition - ( (long)sliceIdx << sliceShift ) ) ); } } @@ -670,12 +693,13 @@ public class MappedByteBufferInputStream extends InputStream { @Override public final synchronized int read() throws IOException { checkOpen(); - if ( ! slice( currSlice ).hasRemaining() ) { - if ( !nextSlice() ) { + ByteBuffer slice = currentSlice(); + if ( !slice.hasRemaining() ) { + if ( null == ( slice = nextSlice() ) ) { return -1; } } - return slices[ currSlice ].get() & 0xFF; + return slice.get() & 0xFF; } @Override @@ -683,10 +707,54 @@ public class MappedByteBufferInputStream extends InputStream { checkOpen(); if (b == null) { throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { + } else if( off < 0 || + len < 0 || + off > b.length || + off + len > b.length || + off + len < 0 + ) { throw new IndexOutOfBoundsException("offset "+off+", length "+len+", b.length "+b.length); + } else if ( 0 == len ) { + return 0; } - if ( 0 == len ) { + final long totalRem = remaining(); + if ( 0 == totalRem ) { + return -1; + } + final int maxLen = (int)Math.min( totalRem, len ); + int read = 0; + while( read < maxLen ) { + ByteBuffer slice = currentSlice(); + int currRem = slice.remaining(); + if ( 0 == currRem ) { + if ( null == ( slice = nextSlice() ) ) { + throw new InternalError("Unexpected EOT"); + } + currRem = slice.remaining(); + } + final int currLen = Math.min( maxLen - read, currRem ); + slice.get( b, off + read, currLen ); + read += currLen; + } + return maxLen; + } + + /** + * Perform similar to {@link #read(byte[], int, int)} + * with {@link ByteBuffer} instead of byte array. + * @param b the {@link ByteBuffer} sink, data is written at current {@link ByteBuffer#position()} + * @param len the number of bytes to read + * @return the number of bytes read, -1 for EOS + * @throws IOException if a buffer slice operation failed or stream has been {@link #close() closed}. + */ + // @Override + public final synchronized int read(final ByteBuffer b, final int len) throws IOException { + checkOpen(); + if (b == null) { + throw new NullPointerException(); + } else if (len < 0 || len > b.remaining()) { + throw new IndexOutOfBoundsException("length "+len+", b "+b); + } else if ( 0 == len ) { return 0; } final long totalRem = remaining(); @@ -696,15 +764,33 @@ public class MappedByteBufferInputStream extends InputStream { final int maxLen = (int)Math.min( totalRem, len ); int read = 0; while( read < maxLen ) { - int currRem = slice( currSlice ).remaining(); + ByteBuffer slice = currentSlice(); + int currRem = slice.remaining(); if ( 0 == currRem ) { - if ( !nextSlice() ) { - throw new InternalError("XX"); + if ( null == ( slice = nextSlice() ) ) { + throw new InternalError("Unexpected EOT"); + } + currRem = slice.remaining(); + } + final int currLen = Math.min( maxLen - read, currRem ); + if( slice.hasArray() && b.hasArray() ) { + System.arraycopy(slice.array(), slice.arrayOffset() + slice.position(), + b.array(), b.arrayOffset() + b.position(), + currLen); + slice.position( slice.position() + currLen ); + b.position( b.position() + currLen ); + } else if( currLen == currRem ) { + b.put(slice); + } else { + final int _limit = slice.limit(); + slice.limit(currLen); + try { + b.put(slice); + } finally { + slice.limit(_limit); } - currRem = slice( currSlice ).remaining(); } - slices[ currSlice ].get( b, off + read, Math.min( maxLen - read, currRem ) ); - read += Math.min( maxLen - read, currRem ); + read += currLen; } return maxLen; } diff --git a/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java b/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java index 498e9f7..f84e6c2 100644 --- a/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java +++ b/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java @@ -29,12 +29,53 @@ package com.jogamp.common.nio; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import com.jogamp.common.nio.MappedByteBufferInputStream.CacheMode; +import com.jogamp.common.nio.MappedByteBufferInputStream.FileResizeOp; + +/** + * An {@link OutputStream} implementation based on an underlying {@link FileChannel}'s memory mapped {@link ByteBuffer}. + * <p> + * Implementation is based on {@link MappedByteBufferInputStream}, using it as its parent instance. + * </p> + * <p> + * An instance maybe created via its parent {@link MappedByteBufferInputStream#getOutputStream(FileResizeOp)} + * or directly {@link #MappedByteBufferOutputStream(FileChannel, MapMode, CacheMode, int, FileResizeOp)}. + * </p> + * @since 2.3.0 + */ public class MappedByteBufferOutputStream extends OutputStream { private final MappedByteBufferInputStream parent; - MappedByteBufferOutputStream(final MappedByteBufferInputStream stream) throws IOException { - this.parent = stream; + MappedByteBufferOutputStream(final MappedByteBufferInputStream parent, + final FileResizeOp fileResizeOp) throws IOException { + if( FileChannel.MapMode.READ_ONLY == parent.getMapMode() ) { + throw new IOException("FileChannel map-mode is read-only"); + } + this.parent = parent; + this.parent.setFileResizeOp(fileResizeOp); + } + + /** + * Creates a new instance using the given {@link FileChannel}. + * <p> + * The {@link ByteBuffer} slices will be mapped lazily at first usage. + * </p> + * @param fileChannel the file channel to be mapped lazily. + * @param mmode the map mode, default is {@link FileChannel.MapMode#READ_WRITE}. + * @param cmode the caching mode, default is {@link MappedByteBufferInputStream.CacheMode#FLUSH_PRE_SOFT}. + * @param sliceShift the pow2 slice size, default is {@link MappedByteBufferInputStream#DEFAULT_SLICE_SHIFT}. + * @param fileResizeOp {@link MappedByteBufferInputStream.FileResizeOp} as described on {@link MappedByteBufferInputStream#setFileResizeOp(FileResizeOp)}. + * @throws IOException + */ + public MappedByteBufferOutputStream(final FileChannel fileChannel, + final FileChannel.MapMode mmode, + final CacheMode cmode, + final int sliceShift, final FileResizeOp fileResizeOp) throws IOException { + this(new MappedByteBufferInputStream(fileChannel, mmode, cmode, sliceShift, fileChannel.size(), 0), fileResizeOp); } /** @@ -103,16 +144,18 @@ public class MappedByteBufferOutputStream extends OutputStream { if ( totalRem < 1 ) { // grow if required parent.setLength( parent.length() + 1 ); } - if ( ! parent.slice( parent.currSlice ).hasRemaining() ) { - if ( !parent.nextSlice() ) { + ByteBuffer slice = parent.currentSlice(); + final int currRem = slice.remaining(); + if ( 0 == currRem ) { + if ( null == ( slice = parent.nextSlice() ) ) { if( MappedByteBufferInputStream.DEBUG ) { - System.err.println("EOT write: "+parent.slices[ parent.currSlice ]); + System.err.println("EOT write: "+parent.currentSlice()); parent.dbgDump("EOT write:", System.err); } - throw new IOException("EOT"); + throw new IOException("EOT"); // 'end-of-tape' } } - parent.slices[ parent.currSlice ].put( (byte)(b & 0xFF) ); + slice.put( (byte)(b & 0xFF) ); } @Override @@ -121,8 +164,8 @@ public class MappedByteBufferOutputStream extends OutputStream { if (b == null) { throw new NullPointerException(); } else if( off < 0 || - off > b.length || len < 0 || + off > b.length || off + len > b.length || off + len < 0 ) { @@ -136,21 +179,132 @@ public class MappedByteBufferOutputStream extends OutputStream { } int written = 0; while( written < len ) { - int currRem = parent.slice( parent.currSlice ).remaining(); + ByteBuffer slice = parent.currentSlice(); + int currRem = slice.remaining(); if ( 0 == currRem ) { - if ( !parent.nextSlice() ) { + if ( null == ( slice = parent.nextSlice() ) ) { if( MappedByteBufferInputStream.DEBUG ) { System.err.println("EOT write: offset "+off+", length "+len+", b.length "+b.length); System.err.println("EOT write: written "+written+" / "+len+", currRem "+currRem); - System.err.println("EOT write: "+parent.slices[ parent.currSlice ]); + System.err.println("EOT write: "+parent.currentSlice()); parent.dbgDump("EOT write:", System.err); } - throw new InternalError("EOT"); + throw new InternalError("EOT"); // 'end-of-tape' } - currRem = parent.slice( parent.currSlice ).remaining(); + currRem = slice.remaining(); + } + final int currLen = Math.min( len - written, currRem ); + slice.put( b, off + written, currLen ); + written += currLen; + } + } + + /** + * Perform similar to {@link #write(byte[], int, int)} + * with {@link ByteBuffer} instead of byte array. + * @param b the {@link ByteBuffer} source, data is read from current {@link ByteBuffer#position()} + * @param len the number of bytes to write + * @throws IOException if a buffer slice operation failed or stream has been {@link #close() closed}. + */ + // @Override + public final synchronized void write(final ByteBuffer b, final int len) throws IOException { + parent.checkOpen(); + if (b == null) { + throw new NullPointerException(); + } else if (len < 0 || len > b.remaining()) { + throw new IndexOutOfBoundsException("length "+len+", b "+b); + } else if( 0 == len ) { + return; + } + final long totalRem = parent.remaining(); + if ( totalRem < len ) { // grow if required + parent.setLength( parent.length() + len - totalRem ); + } + int written = 0; + while( written < len ) { + ByteBuffer slice = parent.currentSlice(); + int currRem = slice.remaining(); + if ( 0 == currRem ) { + if ( null == ( slice = parent.nextSlice() ) ) { + if( MappedByteBufferInputStream.DEBUG ) { + System.err.println("EOT write: length "+len+", b "+b); + System.err.println("EOT write: written "+written+" / "+len+", currRem "+currRem); + System.err.println("EOT write: "+parent.currentSlice()); + parent.dbgDump("EOT write:", System.err); + } + throw new InternalError("EOT"); // 'end-of-tape' + } + currRem = slice.remaining(); + } + final int currLen = Math.min( len - written, currRem ); + + if( slice.hasArray() && b.hasArray() ) { + System.arraycopy(b.array(), b.arrayOffset() + b.position(), + slice.array(), slice.arrayOffset() + slice.position(), + currLen); + b.position( b.position() + currLen ); + slice.position( slice.position() + currLen ); + } else if( currLen == currRem ) { + slice.put(b); + } else { + final int _limit = b.limit(); + b.limit(currLen); + try { + slice.put(b); + } finally { + b.limit(_limit); + } + } + written += currLen; + } + } + + /** + * Perform similar to {@link #write(ByteBuffer, int)} + * with {@link MappedByteBufferInputStream} instead of byte array. + * <p> + * Method directly copies memory mapped {@link ByteBuffer}'ed data + * from the given input stream to this stream without extra data copy. + * </p> + * @param b the {@link ByteBuffer} source, data is read from current {@link MappedByteBufferInputStream#position()} + * @param len the number of bytes to write + * @throws IOException if a buffer slice operation failed or stream has been {@link #close() closed}. + */ + // @Override + public final synchronized void write(final MappedByteBufferInputStream b, final long len) throws IOException { + parent.checkOpen(); + if (b == null) { + throw new NullPointerException(); + } else if (len < 0 || len > b.remaining()) { + throw new IndexOutOfBoundsException("length "+len+", b "+b); + } else if( 0 == len ) { + return; + } + final long totalRem = parent.remaining(); + if ( totalRem < len ) { // grow if required + parent.setLength( parent.length() + len - totalRem ); + } + long written = 0; + while( written < len ) { + ByteBuffer slice = parent.currentSlice(); + int currRem = slice.remaining(); + if ( 0 == currRem ) { + if ( null == ( slice = parent.nextSlice() ) ) { + if( MappedByteBufferInputStream.DEBUG ) { + System.err.println("EOT write: length "+len+", b "+b); + System.err.println("EOT write: written "+written+" / "+len+", currRem "+currRem); + System.err.println("EOT write: "+parent.currentSlice()); + parent.dbgDump("EOT write:", System.err); + } + throw new InternalError("EOT"); // 'end-of-tape' + } + currRem = slice.remaining(); + } + final int currLen = b.read(slice, (int)Math.min( len - written, currRem )); + if( 0 > currLen ) { + throw new InternalError("Unexpected InputStream EOT"); // 'end-of-tape' } - parent.slices[ parent.currSlice ].put( b, off + written, Math.min( len - written, currRem ) ); - written += Math.min( len - written, currRem ); + written += currLen; } } } diff --git a/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java b/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java new file mode 100644 index 0000000..3442159 --- /dev/null +++ b/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java @@ -0,0 +1,177 @@ +/** + * Copyright 2014 JogAmp Community. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY JogAmp Community ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JogAmp Community OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are those of the + * authors and should not be interpreted as representing official policies, either expressed + * or implied, of JogAmp Community. + */ +package com.jogamp.common.nio; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; + +import org.junit.Assert; +import org.junit.Test; + +import com.jogamp.junit.util.JunitTracer; + +import org.junit.FixMethodOrder; +import org.junit.runners.MethodSorters; + +/** + * Testing {@link MappedByteBufferInputStream} and {@link MappedByteBufferOutputStream} + * direct stream to stream copy via mapped buffers. + */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestByteBufferCopyStream extends JunitTracer { + + static void testImpl(final String srcFileName, final long size, + final MappedByteBufferInputStream.CacheMode srcCacheMode, final int srcSliceShift, + final String dstFileName, + final MappedByteBufferInputStream.CacheMode dstCacheMode, final int dstSliceShift ) throws IOException { + final Runtime runtime = Runtime.getRuntime(); + final long[] usedMem0 = { 0 }; + final long[] freeMem0 = { 0 }; + final long[] usedMem1 = { 0 }; + final long[] freeMem1 = { 0 }; + final String prefix = "test "+String.format(TestByteBufferInputStream.PrintPrecision+" MiB", size/TestByteBufferInputStream.MIB); + TestByteBufferInputStream.dumpMem(prefix+" before", runtime, -1, -1, usedMem0, freeMem0 ); + + final File srcFile = new File(srcFileName); + srcFile.delete(); + srcFile.createNewFile(); + srcFile.deleteOnExit(); + + final RandomAccessFile input; + { + final RandomAccessFile _input = new RandomAccessFile(srcFile, "rw"); + _input.setLength(size); + _input.close(); + input = new RandomAccessFile(srcFile, "r"); + } + final MappedByteBufferInputStream mis = new MappedByteBufferInputStream(input.getChannel(), + FileChannel.MapMode.READ_ONLY, + srcCacheMode, + srcSliceShift); + Assert.assertEquals(size, input.length()); + Assert.assertEquals(size, mis.length()); + Assert.assertEquals(0, mis.position()); + Assert.assertEquals(size, mis.remaining()); + + final File dstFile = new File(dstFileName); + dstFile.delete(); + dstFile.createNewFile(); + dstFile.deleteOnExit(); + final RandomAccessFile output = new RandomAccessFile(dstFile, "rw"); + final MappedByteBufferInputStream.FileResizeOp szOp = new MappedByteBufferInputStream.FileResizeOp() { + @Override + public void setLength(final long newSize) throws IOException { + output.setLength(newSize); + } + }; + final MappedByteBufferOutputStream mos = new MappedByteBufferOutputStream(output.getChannel(), + FileChannel.MapMode.READ_WRITE, + dstCacheMode, + srcSliceShift, szOp); + Assert.assertEquals(0, output.length()); + Assert.assertEquals(0, mos.length()); + Assert.assertEquals(0, mos.position()); + Assert.assertEquals(0, mos.remaining()); + + mos.write(mis, mis.remaining()); + + Assert.assertEquals(size, input.length()); + Assert.assertEquals(size, output.length()); + Assert.assertEquals(size, mis.length()); + Assert.assertEquals(size, mos.length()); + Assert.assertEquals(size, mis.position()); + Assert.assertEquals(size, mos.position()); + Assert.assertEquals(0, mis.remaining()); + Assert.assertEquals(0, mos.remaining()); + + mos.close(); + mis.close(); + input.close(); + output.close(); + srcFile.delete(); + dstFile.delete(); + TestByteBufferInputStream.dumpMem(prefix+" after ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); + System.gc(); + try { + Thread.sleep(500); + } catch (final InterruptedException e) { } + TestByteBufferInputStream.dumpMem(prefix+" gc'ed ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); + } + + @Test + public void test00() throws IOException { + final int srcSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; + final int dstSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; + final long size = 3L * ( 1L << 30 ); // 3 GiB + testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, srcSliceShift, + "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, dstSliceShift ); + } + + @Test + public void test01() throws IOException { + final int srcSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; + final int dstSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; + final long size = 3L * ( 1L << 30 ); // 3 GiB + testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + } + + @Test + public void test02() throws IOException { + final int srcSliceShift = 28; // 256M bytes per slice + final int dstSliceShift = 28; // 256M bytes per slice + final long size = 3L * ( 1L << 30 ); // 3 GiB + testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + } + + @Test + public void test11() throws IOException { + final int srcSliceShift = 28; // 256M bytes per slice + final int dstSliceShift = 27; // 128M bytes per slice + final long size = 3L * ( 1L << 30 ); // 3 GiB + testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + } + + @Test + public void test12() throws IOException { + final int srcSliceShift = 27; // 128M bytes per slice + final int dstSliceShift = 28; // 256M bytes per slice + final long size = 3L * ( 1L << 30 ); // 3 GiB + testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + } + + public static void main(final String args[]) throws IOException { + final String tstname = TestByteBufferCopyStream.class.getName(); + org.junit.runner.JUnitCore.main(tstname); + } +} diff --git a/src/junit/com/jogamp/common/nio/TestByteBufferInputStream.java b/src/junit/com/jogamp/common/nio/TestByteBufferInputStream.java index 195bef3..61961a3 100644 --- a/src/junit/com/jogamp/common/nio/TestByteBufferInputStream.java +++ b/src/junit/com/jogamp/common/nio/TestByteBufferInputStream.java @@ -84,15 +84,15 @@ public class TestByteBufferInputStream extends JunitTracer { static final String fileTwoPlusGiB = "./testTwoPlusGiB.bin" ; static final String fileOut = "./testOut.bin" ; - static final String printPrecision = "%8.3f"; - static final double mib = 1024.0*1024.0; + public static final String PrintPrecision = "%8.3f"; + public static final double MIB = 1024.0*1024.0; @BeforeClass public static void setup() throws IOException { final Runtime runtime = Runtime.getRuntime(); - System.err.printf("Total Memory : "+printPrecision+" MiB%n", runtime.totalMemory() / mib); - System.err.printf("Max Memory : "+printPrecision+" MiB%n", runtime.maxMemory() / mib); + System.err.printf("Total Memory : "+PrintPrecision+" MiB%n", runtime.totalMemory() / MIB); + System.err.printf("Max Memory : "+PrintPrecision+" MiB%n", runtime.maxMemory() / MIB); setup(fileHalfMiB, halfMiB); setup(fileOneMiB, oneMiB); @@ -189,7 +189,7 @@ public class TestByteBufferInputStream extends JunitTracer { final long[] usedMem1 = { 0 }; final long[] freeMem1 = { 0 }; - final String prefix = "test #"+iter+" "+String.format(printPrecision+" MiB", expSize/mib); + final String prefix = "test #"+iter+" "+String.format(PrintPrecision+" MiB", expSize/MIB); System.err.printf("%s: mode %-5s, bufferSize %9d: BEGIN%n", prefix, srcType.toString(), bufferSize); dumpMem(prefix+" before", runtime, -1, -1, usedMem0, freeMem0 ); @@ -323,17 +323,17 @@ public class TestByteBufferInputStream extends JunitTracer { } } - static void dumpMem(final String pre, - final Runtime runtime, final long usedMem0, - final long freeMem0, final long[] usedMemN, - final long[] freeMemN ) + public static void dumpMem(final String pre, + final Runtime runtime, final long usedMem0, + final long freeMem0, final long[] usedMemN, + final long[] freeMemN ) { usedMemN[0] = runtime.totalMemory() - runtime.freeMemory(); freeMemN[0] = runtime.freeMemory(); - System.err.printf("%s Used Memory : "+printPrecision, pre, usedMemN[0] / mib); + System.err.printf("%s Used Memory : "+PrintPrecision, pre, usedMemN[0] / MIB); if( 0 < usedMem0 ) { - System.err.printf(", delta "+printPrecision, (usedMemN[0]-usedMem0) / mib); + System.err.printf(", delta "+PrintPrecision, (usedMemN[0]-usedMem0) / MIB); } System.err.println(" MiB"); /** diff --git a/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java b/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java index 10d7f50..c06dbe8 100644 --- a/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java +++ b/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java @@ -47,9 +47,9 @@ import org.junit.runners.MethodSorters; public class TestByteBufferOutputStream extends JunitTracer { static void testImpl(final String fname, - final byte[] payLoad, final long payLoadOffset, final long postPayLoadFiller, - final byte[] endBytes, - final int sliceShift) + final byte[] payLoad, final long payLoadOffset, final long postPayLoadFiller, + final byte[] endBytes, + final int sliceShift) throws IOException { final File file = new File(fname); |