From a7a3d5ab98ee0ad33fdef50bf081afeb8295ebe4 Mon Sep 17 00:00:00 2001 From: Sven Gothel Date: Fri, 3 Oct 2014 03:12:42 +0200 Subject: MappedByteBuffer*Stream: - Validate active and GC'ed mapped-buffer count in cleanAllSlices() via close() .. - Fix missing unmapping last buffer in notifyLengthChangeImpl(), branch criteria was off by one. - cleanSlice(..) now also issues cleanBuffer(..) on the GC'ed entry, hence if WeakReference is still alive, enforce it's release. - cleanBuffer(..) reverts FLUSH_PRE_HARD -> FLUSH_PRE_SOFT in case of an error. - flush() -> flush(boolean metaData) to expose FileChannel.force(metaData). - Add synchronous mode, flushing/syncing the mapped buffers when in READ_WRITE mapping mode and issue FileChannel.force() if not READ_ONLY. Above is implemented via flush()/flushImpl(..) for buffers and FileChannel, as well as in syncSlice(..) for buffers only. flush*()/syncSlice() is covered by: - setLength() - notifyLengthChange*(..) - nextSlice() Always issue flushImpl() in close(). - Windows: Clean all buffers in setLength(), otherwise Windows will report: - Windows: Catch MappedByteBuffer.force() IOException - Optimization of position(..) position(..) is now standalone to allow issuing flushSlice(..) before gathering the new mapped buffer. This shall avoid one extra cache miss. Hence rename positionImpl(..) -> position2(..). - All MappedByteBufferOutputStream.write(..) methods issue syncSlice(..) on the last written current slice to ensure new 'synchronous' mode is honored. +++ Unit tests: - Ensure test files are being deleted - TestByteBufferCopyStream: Reduced test file size to more sensible values. - --- .../common/nio/MappedByteBufferInputStream.java | 281 ++++++++++++++----- .../common/nio/MappedByteBufferOutputStream.java | 49 +++- .../common/nio/TestByteBufferCopyStream.java | 105 +++++--- .../common/nio/TestByteBufferOutputStream.java | 298 +++++++++++---------- src/junit/com/jogamp/common/util/TestIOUtil01.java | 37 ++- 5 files changed, 517 insertions(+), 253 deletions(-) (limited to 'src') diff --git a/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java b/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java index 5f91f64..1d4d78a 100644 --- a/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java +++ b/src/java/com/jogamp/common/nio/MappedByteBufferInputStream.java @@ -29,11 +29,13 @@ package com.jogamp.common.nio; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.io.RandomAccessFile; import java.lang.ref.WeakReference; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.security.AccessController; @@ -156,6 +158,8 @@ public class MappedByteBufferInputStream extends InputStream { private ByteBuffer[] slices; private WeakReference[] slices2GC; private long totalSize; + private int slicesEntries, slices2GCEntries; + private boolean synchronous; private int refCount; @@ -169,6 +173,23 @@ public class MappedByteBufferInputStream extends InputStream { private long mark; final void dbgDump(final String prefix, final PrintStream out) { + int _slicesEntries = 0; + for(int i=0; i ref = slices2GC[i]; + if( null != ref ) { + _slices2GCEntries++; + if( null != ref.get() ) { + _slices2GCAliveEntries++; + } + } + } long fcSz = 0, pos = 0, rem = 0; try { fcSz = fc.size(); @@ -187,7 +208,9 @@ public class MappedByteBufferInputStream extends InputStream { out.println(prefix+" refCount "+refCount+", fcSize "+fcSz+", totalSize "+totalSize); out.println(prefix+" position "+pos+", remaining "+rem); out.println(prefix+" mmode "+mmode+", cmode "+cmode+", fileResizeOp "+fileResizeOp); - out.println(prefix+" slice "+sliceIdx+" / "+sliceCount+" ("+sliceCount2+")"); + out.println(prefix+" slice "+sliceIdx+" / "+sliceCount+" ("+sliceCount2+"), synchronous "+synchronous); + out.println(prefix+" mapped "+slicesEntries+" / "+_slicesEntries); + out.println(prefix+" GC-queue "+slices2GCEntries+" / "+_slices2GCEntries+" (alive "+_slices2GCAliveEntries+")"); out.println(prefix+" sliceShift "+sliceShift+" -> "+(1L << sliceShift)); } @@ -203,7 +226,7 @@ public class MappedByteBufferInputStream extends InputStream { // trigger notifyLengthChange this.totalSize = -1; this.sliceCount = 0; - notifyLengthChange(totalSize); + notifyLengthChange( totalSize ); this.refCount = 1; this.cleanerInit = false; @@ -263,6 +286,28 @@ public class MappedByteBufferInputStream extends InputStream { this(fileChannel, FileChannel.MapMode.READ_ONLY, CacheMode.FLUSH_PRE_SOFT, DEFAULT_SLICE_SHIFT); } + /** + * Enable or disable synchronous mode. + *

+ * If synchronous mode is enabled, mapped buffers will be {@link #flush(boolean) flushed} + * if {@link #notifyLengthChange(long) resized}, written to or {@link #close() closing} in {@link FileChannel.MapMode#READ_WRITE read-write} mapping mode. + *

+ *

+ * If synchronous mode is enabled, {@link FileChannel#force(boolean)} is issued + * if {@link #setLength(long) resizing} or {@link #close() closing} and not in {@link FileChannel.MapMode#READ_ONLY read-only} mapping mode. + *

+ * @param s {@code true} to enable synchronous mode + */ + public final synchronized void setSynchronous(final boolean s) { + synchronous = s; + } + /** + * Return {@link #setSynchronous(boolean) synchronous mode}. + */ + public final synchronized boolean getSynchronous() { + return synchronous ; + } + final synchronized void checkOpen() throws IOException { if( 0 == refCount ) { throw new IOException("stream closed"); @@ -274,16 +319,15 @@ public class MappedByteBufferInputStream extends InputStream { if( 0 < refCount ) { refCount--; if( 0 == refCount ) { - for(int i=0; i * User must have a {@link FileResizeOp} {@link #setFileResizeOp(FileResizeOp) registered} before. *

+ *

+ * Implementation calls {@link #notifyLengthChange(long)} after {@link FileResizeOp#setLength(long)}. + *

* @param newTotalSize the new total size * @throws IOException if no {@link FileResizeOp} has been {@link #setFileResizeOp(FileResizeOp) registered} * or if a buffer slice operation failed */ public final synchronized void setLength(final long newTotalSize) throws IOException { + final long currentPosition; + if( 0 != newTotalSize && totalSize != newTotalSize ) { + currentPosition = position(); + } else { + currentPosition = -1L; + } if( fc.size() != newTotalSize ) { + if( Platform.OSType.WINDOWS == Platform.getOSType() ) { + // On Windows, we have to close all mapped slices. + // Otherwise we will receive: + // java.io.IOException: The requested operation cannot be performed on a file with a user-mapped section open + // at java.io.RandomAccessFile.setLength(Native Method) + cleanAllSlices( synchronous ); + } fileResizeOp.setLength(newTotalSize); + if( synchronous ) { + // buffers will be synchronized in notifyLengthChangeImpl(..) + flushImpl( true /* metaData */, false /* syncBuffer */); + } } - notifyLengthChange(newTotalSize); + notifyLengthChangeImpl(newTotalSize, currentPosition); } /** @@ -328,6 +392,9 @@ public class MappedByteBufferInputStream extends InputStream { * @throws IOException if a buffer slice operation failed */ public final synchronized void notifyLengthChange(final long newTotalSize) throws IOException { + notifyLengthChangeImpl(newTotalSize, -1L); + } + private final synchronized void notifyLengthChangeImpl(final long newTotalSize, final long currentPosition) throws IOException { /* if( DEBUG ) { System.err.println("notifyLengthChange.0: "+totalSize+" -> "+newTotalSize); dbgDump("notifyLengthChange.0:", System.err); @@ -337,11 +404,7 @@ public class MappedByteBufferInputStream extends InputStream { return; } else if( 0 == newTotalSize ) { // ZERO - ensure one entry avoiding NULL checks - if( null != slices ) { - for(int i=0; i[] newSlices2GC = new WeakReference[ 1 ]; slices2GC = newSlices2GC; @@ -352,7 +415,7 @@ public class MappedByteBufferInputStream extends InputStream { mark = -1; sliceIdx = 0; } else { - final long prePosition = position(); + final long prePosition = 0 <= currentPosition ? currentPosition : position(); final long sliceSize = 1L << sliceShift; final int newSliceCount = (int)( ( newTotalSize + ( sliceSize - 1 ) ) / sliceSize ); @@ -360,11 +423,13 @@ public class MappedByteBufferInputStream extends InputStream { final WeakReference[] newSlices2GC = new WeakReference[ newSliceCount ]; final ByteBuffer[] newSlices = new ByteBuffer[ newSliceCount ]; final int copySliceCount = Math.min(newSliceCount, sliceCount-1); // drop last (resize) - if( 0 < copySliceCount ) { - System.arraycopy(slices2GC, 0, newSlices2GC, 0, copySliceCount); - System.arraycopy(slices, 0, newSlices, 0, copySliceCount); + if( 0 <= copySliceCount ) { + if( 0 < copySliceCount ) { + System.arraycopy(slices2GC, 0, newSlices2GC, 0, copySliceCount); + System.arraycopy(slices, 0, newSlices, 0, copySliceCount); + } for(int i=copySliceCount; i clipped position (set currSlice and re-map/-pos buffer) + position2( Math.min(prePosition, newTotalSize) ); // -> clipped position (set currSlice and re-map/-pos buffer) } /* if( DEBUG ) { System.err.println("notifyLengthChange.X: "+slices[currSlice]); @@ -383,16 +448,34 @@ public class MappedByteBufferInputStream extends InputStream { } /** - * + * Similar to {@link OutputStream#flush()}, synchronizes all mapped buffers + * from local storage via {@link MappedByteBuffer#force()} + * as well as the {@link FileChannel#force(boolean)} w/o {@code metaData}. + * @param metaData TODO * @throws IOException if this stream has been {@link #close() closed}. */ - public final synchronized void flush() throws IOException { + public final synchronized void flush(final boolean metaData) throws IOException { checkOpen(); - if( mmode != FileChannel.MapMode.READ_ONLY ) { - fc.force(true); + flushImpl(metaData, true); + } + private final synchronized void flushImpl(final boolean metaData, final boolean syncBuffer) throws IOException { + if( FileChannel.MapMode.READ_ONLY != mmode ) { + if( syncBuffer && FileChannel.MapMode.READ_WRITE == mmode ) { + for(int i=0; i ref = slices2GC[i]; + if( null != ref ) { + syncSlice(ref.get(), true); + } + } + } + fc.force(metaData); } } + /** * Returns a new MappedByteBufferOutputStream instance sharing * all resources of this input stream, including all buffer slices. @@ -422,23 +505,28 @@ public class MappedByteBufferInputStream extends InputStream { * @throws IOException if a buffer slice operation failed. */ public final synchronized ByteBuffer currentSlice() throws IOException { - if ( null != slices[sliceIdx] ) { - return slices[sliceIdx]; + final ByteBuffer s0 = slices[sliceIdx]; + if ( null != s0 ) { + return s0; } else { if( CacheMode.FLUSH_PRE_SOFT == cmode ) { final WeakReference ref = slices2GC[sliceIdx]; if( null != ref ) { final ByteBuffer mbb = ref.get(); slices2GC[sliceIdx] = null; + slices2GCEntries--; if( null != mbb ) { slices[sliceIdx] = mbb; + slicesEntries++; return mbb; } } } final long pos = (long)sliceIdx << sliceShift; - slices[sliceIdx] = fc.map(mmode, pos, Math.min(1L << sliceShift, totalSize - pos)); - return slices[sliceIdx]; + final MappedByteBuffer s1 = fc.map(mmode, pos, Math.min(1L << sliceShift, totalSize - pos)); + slices[sliceIdx] = s1; + slicesEntries++; + return s1; } } @@ -453,9 +541,7 @@ public class MappedByteBufferInputStream extends InputStream { */ public final synchronized ByteBuffer nextSlice() throws IOException { if ( sliceIdx < sliceCount - 1 ) { - if( CacheMode.FLUSH_NONE != cmode ) { - flushSlice(sliceIdx); - } + flushSlice(sliceIdx, synchronous); sliceIdx++; final ByteBuffer slice = currentSlice(); slice.position( 0 ); @@ -465,46 +551,109 @@ public class MappedByteBufferInputStream extends InputStream { } } - private synchronized void flushSlice(final int i) throws IOException { + synchronized void syncSlice(final ByteBuffer s) throws IOException { + syncSlice(s, synchronous); + } + synchronized void syncSlice(final ByteBuffer s, final boolean syncBuffer) throws IOException { + if( syncBuffer && null != s && FileChannel.MapMode.READ_WRITE == mmode ) { + try { + ((MappedByteBuffer)s).force(); + } catch( final Throwable t ) { + // On Windows .. this may happen, like: + // java.io.IOException: The process cannot access the file because another process has locked a portion of the file + // at java.nio.MappedByteBuffer.force0(Native Method) + // at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) + if( DEBUG ) { + System.err.println("Caught "+t.getMessage()); + t.printStackTrace(); + } + } + } + } + private synchronized void flushSlice(final int i, final boolean syncBuffer) throws IOException { final ByteBuffer s = slices[i]; if ( null != s ) { - slices[i] = null; // GC a slice is enough - if( CacheMode.FLUSH_PRE_HARD == cmode ) { - if( !cleanBuffer(s) ) { - cmode = CacheMode.FLUSH_PRE_SOFT; + if( CacheMode.FLUSH_NONE != cmode ) { + slices[i] = null; // trigger slice GC + slicesEntries--; + if( CacheMode.FLUSH_PRE_HARD == cmode ) { + if( !cleanBuffer(s, syncBuffer) ) { + // buffer already synced in cleanBuffer(..) if requested + slices2GC[i] = new WeakReference(s); + slices2GCEntries++; + } + } else { + syncSlice(s, syncBuffer); slices2GC[i] = new WeakReference(s); + slices2GCEntries++; } } else { - slices2GC[i] = new WeakReference(s); + syncSlice(s, syncBuffer); } } } - private synchronized void cleanSlice(final int i) { - final ByteBuffer s = slices[i]; - if( null != s ) { + private synchronized void cleanAllSlices(final boolean syncBuffers) throws IOException { + if( null != slices ) { + for(int i=0; i ref = slices2GC[i]; + slices2GC[i] = null; + if( null != ref ) { + slices2GCEntries--; + s2 = ref.get(); + } else { + s2 = null; + } + } + if( null != s1 ) { slices[i] = null; - cleanBuffer(s); + slicesEntries--; + cleanBuffer(s1, syncBuffer); + if( null != s2 ) { + throw new InternalError("XXX"); + } + } else if( null != s2 ) { + cleanBuffer(s2, syncBuffer); } - slices2GC[i] = null; } - private synchronized boolean cleanBuffer(final ByteBuffer mbb) { + private synchronized boolean cleanBuffer(final ByteBuffer mbb, final boolean syncBuffer) throws IOException { if( !cleanerInit ) { initCleaner(mbb); } - if ( !hasCleaner || !mbb.isDirect() ) { + syncSlice(mbb, syncBuffer); + if( !mbb.isDirect() ) { return false; } - try { - cClean.invoke(mbbCleaner.invoke(mbb)); - return true; - } catch(final Throwable t) { - hasCleaner = false; - if( DEBUG ) { - System.err.println("Caught "+t.getMessage()); - t.printStackTrace(); + boolean res = false; + if ( hasCleaner ) { + try { + cClean.invoke(mbbCleaner.invoke(mbb)); + res = true; + } catch(final Throwable t) { + hasCleaner = false; + if( DEBUG ) { + System.err.println("Caught "+t.getMessage()); + t.printStackTrace(); + } } - return false; } + if( !res && CacheMode.FLUSH_PRE_HARD == cmode ) { + cmode = CacheMode.FLUSH_PRE_SOFT; + } + return res; } private synchronized void initCleaner(final ByteBuffer bb) { final Method[] _mbbCleaner = { null }; @@ -618,13 +767,25 @@ public class MappedByteBufferInputStream extends InputStream { throw new IllegalArgumentException("new position "+newPosition+" not within [0.."+totalSize+"]"); } final int preSlice = sliceIdx; - positionImpl( newPosition ); - if( CacheMode.FLUSH_NONE != cmode && preSlice != sliceIdx) { - flushSlice(preSlice); + + if ( totalSize == newPosition ) { + // EOF, pos == maxPos + 1 + sliceIdx = Math.max(0, sliceCount - 1); // handle zero size + if( preSlice != sliceIdx ) { + flushSlice(preSlice, synchronous); + } + final ByteBuffer s = currentSlice(); + s.position( s.capacity() ); + } else { + sliceIdx = (int)( newPosition >>> sliceShift ); + if( preSlice != sliceIdx ) { + flushSlice(preSlice, synchronous); + } + currentSlice().position( (int)( newPosition - ( (long)sliceIdx << sliceShift ) ) ); } return this; } - private final synchronized void positionImpl( final long newPosition ) throws IOException { + private final synchronized void position2( final long newPosition ) throws IOException { if ( totalSize == newPosition ) { // EOF, pos == maxPos + 1 sliceIdx = Math.max(0, sliceCount - 1); // handle zero size diff --git a/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java b/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java index f84e6c2..8adf0e4 100644 --- a/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java +++ b/src/java/com/jogamp/common/nio/MappedByteBufferOutputStream.java @@ -78,6 +78,19 @@ public class MappedByteBufferOutputStream extends OutputStream { this(new MappedByteBufferInputStream(fileChannel, mmode, cmode, sliceShift, fileChannel.size(), 0), fileResizeOp); } + /** + * See {@link MappedByteBufferInputStream#setSynchronous(boolean)}. + */ + public final synchronized void setSynchronous(final boolean s) { + parent.setSynchronous(s); + } + /** + * See {@link MappedByteBufferInputStream#getSynchronous()}. + */ + public final synchronized boolean getSynchronous() { + return parent.getSynchronous(); + } + /** * See {@link MappedByteBufferInputStream#setLength(long)}. */ @@ -129,7 +142,15 @@ public class MappedByteBufferOutputStream extends OutputStream { @Override public final synchronized void flush() throws IOException { - parent.flush(); + parent.flush( true /* metaData */); + } + + /** + * See {@link MappedByteBufferInputStream#flush(boolean)}. + */ + // @Override + public final synchronized void flush(final boolean metaData) throws IOException { + parent.flush(metaData); } @Override @@ -156,6 +177,11 @@ public class MappedByteBufferOutputStream extends OutputStream { } } slice.put( (byte)(b & 0xFF) ); + + // sync last buffer (happens only in synchronous mode) + if( null != slice ) { + parent.syncSlice(slice); + } } @Override @@ -178,8 +204,9 @@ public class MappedByteBufferOutputStream extends OutputStream { parent.setLength( parent.length() + len - totalRem ); } int written = 0; + ByteBuffer slice = null; while( written < len ) { - ByteBuffer slice = parent.currentSlice(); + slice = parent.currentSlice(); int currRem = slice.remaining(); if ( 0 == currRem ) { if ( null == ( slice = parent.nextSlice() ) ) { @@ -197,6 +224,10 @@ public class MappedByteBufferOutputStream extends OutputStream { slice.put( b, off + written, currLen ); written += currLen; } + // sync last buffer (happens only in synchronous mode) + if( null != slice ) { + parent.syncSlice(slice); + } } /** @@ -221,8 +252,9 @@ public class MappedByteBufferOutputStream extends OutputStream { parent.setLength( parent.length() + len - totalRem ); } int written = 0; + ByteBuffer slice = null; while( written < len ) { - ByteBuffer slice = parent.currentSlice(); + slice = parent.currentSlice(); int currRem = slice.remaining(); if ( 0 == currRem ) { if ( null == ( slice = parent.nextSlice() ) ) { @@ -257,6 +289,10 @@ public class MappedByteBufferOutputStream extends OutputStream { } written += currLen; } + // sync last buffer (happens only in synchronous mode) + if( null != slice ) { + parent.syncSlice(slice); + } } /** @@ -285,8 +321,9 @@ public class MappedByteBufferOutputStream extends OutputStream { parent.setLength( parent.length() + len - totalRem ); } long written = 0; + ByteBuffer slice = null; while( written < len ) { - ByteBuffer slice = parent.currentSlice(); + slice = parent.currentSlice(); int currRem = slice.remaining(); if ( 0 == currRem ) { if ( null == ( slice = parent.nextSlice() ) ) { @@ -306,5 +343,9 @@ public class MappedByteBufferOutputStream extends OutputStream { } written += currLen; } + // sync last buffer (happens only in synchronous mode) + if( null != slice ) { + parent.syncSlice(slice); + } } } diff --git a/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java b/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java index 3442159..fef26b6 100644 --- a/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java +++ b/src/junit/com/jogamp/common/nio/TestByteBufferCopyStream.java @@ -100,74 +100,107 @@ public class TestByteBufferCopyStream extends JunitTracer { Assert.assertEquals(0, mos.position()); Assert.assertEquals(0, mos.remaining()); - mos.write(mis, mis.remaining()); + OutOfMemoryError oome = null; + IOException ioe = null; - Assert.assertEquals(size, input.length()); - Assert.assertEquals(size, output.length()); - Assert.assertEquals(size, mis.length()); - Assert.assertEquals(size, mos.length()); - Assert.assertEquals(size, mis.position()); - Assert.assertEquals(size, mos.position()); - Assert.assertEquals(0, mis.remaining()); - Assert.assertEquals(0, mos.remaining()); - - mos.close(); - mis.close(); - input.close(); - output.close(); - srcFile.delete(); - dstFile.delete(); - TestByteBufferInputStream.dumpMem(prefix+" after ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); - System.gc(); try { - Thread.sleep(500); - } catch (final InterruptedException e) { } - TestByteBufferInputStream.dumpMem(prefix+" gc'ed ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); + mos.write(mis, mis.remaining()); + + Assert.assertEquals(size, input.length()); + Assert.assertEquals(size, output.length()); + Assert.assertEquals(size, mis.length()); + Assert.assertEquals(size, mos.length()); + Assert.assertEquals(size, mis.position()); + Assert.assertEquals(size, mos.position()); + Assert.assertEquals(0, mis.remaining()); + Assert.assertEquals(0, mos.remaining()); + + } catch (final IOException e) { + if( e.getCause() instanceof OutOfMemoryError ) { + oome = (OutOfMemoryError) e.getCause(); // oops + } else { + ioe = e; + } + } catch (final OutOfMemoryError m) { + oome = m; // oops + } finally { + mos.close(); + mis.close(); + input.close(); + output.close(); + srcFile.delete(); + dstFile.delete(); + TestByteBufferInputStream.dumpMem(prefix+" after ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); + System.gc(); + try { + Thread.sleep(500); + } catch (final InterruptedException e) { } + TestByteBufferInputStream.dumpMem(prefix+" gc'ed ", runtime, usedMem0[0], freeMem0[0], usedMem1, freeMem1 ); + } + if( null != ioe || null != oome ) { + if( null != oome ) { + System.err.printf("%s: OutOfMemoryError.2 %s%n", prefix, oome.getMessage()); + oome.printStackTrace(); + } else { + Assert.assertNull(ioe); + } + } } + /** {@value} */ + static final long halfMiB = 1L << 19; + /** {@value} */ + static final long oneGiB = 1L << 30; + /** {@value} */ + static final long onePlusGiB = oneGiB + halfMiB; + /** {@value} */ + static final long twoGiB = ( 2L << 30 ); + /** {@value} */ + static final long twoPlusGiB = twoGiB + halfMiB; + @Test public void test00() throws IOException { final int srcSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; final int dstSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; - final long size = 3L * ( 1L << 30 ); // 3 GiB - testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, srcSliceShift, - "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, dstSliceShift ); + final long size = twoPlusGiB; + testImpl(getSimpleTestName(".")+"_In.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, srcSliceShift, + getSimpleTestName(".")+"_Out.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_HARD, dstSliceShift ); } @Test public void test01() throws IOException { final int srcSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; final int dstSliceShift = MappedByteBufferInputStream.DEFAULT_SLICE_SHIFT; - final long size = 3L * ( 1L << 30 ); // 3 GiB - testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, - "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + final long size = twoPlusGiB; + testImpl(getSimpleTestName(".")+"_In.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + getSimpleTestName(".")+"_Out.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); } @Test public void test02() throws IOException { final int srcSliceShift = 28; // 256M bytes per slice final int dstSliceShift = 28; // 256M bytes per slice - final long size = 3L * ( 1L << 30 ); // 3 GiB - testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, - "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + final long size = onePlusGiB; + testImpl(getSimpleTestName(".")+"_In.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + getSimpleTestName(".")+"_Out.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); } @Test public void test11() throws IOException { final int srcSliceShift = 28; // 256M bytes per slice final int dstSliceShift = 27; // 128M bytes per slice - final long size = 3L * ( 1L << 30 ); // 3 GiB - testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, - "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + final long size = onePlusGiB; + testImpl(getSimpleTestName(".")+"_In.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + getSimpleTestName(".")+"_Out.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); } @Test public void test12() throws IOException { final int srcSliceShift = 27; // 128M bytes per slice final int dstSliceShift = 28; // 256M bytes per slice - final long size = 3L * ( 1L << 30 ); // 3 GiB - testImpl("./testIn.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, - "./testOut.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); + final long size = onePlusGiB; + testImpl(getSimpleTestName(".")+"_In.bin", size, MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, srcSliceShift, + getSimpleTestName(".")+"_Out.bin", MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, dstSliceShift ); } public static void main(final String args[]) throws IOException { diff --git a/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java b/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java index c06dbe8..f57101e 100644 --- a/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java +++ b/src/junit/com/jogamp/common/nio/TestByteBufferOutputStream.java @@ -51,181 +51,197 @@ public class TestByteBufferOutputStream extends JunitTracer { final byte[] endBytes, final int sliceShift) throws IOException + { + testImpl(fname, payLoad, payLoadOffset, postPayLoadFiller, endBytes, sliceShift, false); + testImpl(fname, payLoad, payLoadOffset, postPayLoadFiller, endBytes, sliceShift, true); + } + static void testImpl(final String fname, + final byte[] payLoad, final long payLoadOffset, final long postPayLoadFiller, + final byte[] endBytes, + final int sliceShift, final boolean synchronous) + throws IOException { final File file = new File(fname); file.delete(); file.createNewFile(); file.deleteOnExit(); - final RandomAccessFile out = new RandomAccessFile(file, "rw"); - final MappedByteBufferInputStream.FileResizeOp szOp = new MappedByteBufferInputStream.FileResizeOp() { - @Override - public void setLength(final long newSize) throws IOException { - out.setLength(newSize); - } - }; - final MappedByteBufferInputStream mis = new MappedByteBufferInputStream(out.getChannel(), - FileChannel.MapMode.READ_WRITE, - MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, - sliceShift); - final MappedByteBufferOutputStream mos = mis.getOutputStream(szOp); - // resize to payLoad start and position to it - mos.setLength(payLoadOffset); - Assert.assertEquals(payLoadOffset, out.length()); - Assert.assertEquals(payLoadOffset, mos.length()); - Assert.assertEquals(0, mos.position()); // no change - mos.position(payLoadOffset); - Assert.assertEquals(payLoadOffset, mos.position()); + try { + final RandomAccessFile out = new RandomAccessFile(file, "rw"); + final MappedByteBufferInputStream.FileResizeOp szOp = new MappedByteBufferInputStream.FileResizeOp() { + @Override + public void setLength(final long newSize) throws IOException { + out.setLength(newSize); + } + }; + final MappedByteBufferInputStream mis = new MappedByteBufferInputStream(out.getChannel(), + FileChannel.MapMode.READ_WRITE, + MappedByteBufferInputStream.CacheMode.FLUSH_PRE_SOFT, + sliceShift); + final MappedByteBufferOutputStream mos = mis.getOutputStream(szOp); + mos.setSynchronous(synchronous); - // mark, write-expand payLoad - mis.mark(1); - mos.write(payLoad); - Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + try { + // resize to payLoad start and position to it + mos.setLength(payLoadOffset); + Assert.assertEquals(payLoadOffset, out.length()); + Assert.assertEquals(payLoadOffset, mos.length()); + Assert.assertEquals(0, mos.position()); // no change + mos.position(payLoadOffset); + Assert.assertEquals(payLoadOffset, mos.position()); - // expand + 1 - mos.setLength(payLoadOffset+payLoad.length+1); - Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); // no change + // mark, write-expand payLoad + mis.mark(1); + mos.write(payLoad); + Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - // expand up-to very end, ahead of write - position to endBytes start - mos.setLength(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length); - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); // no change - mos.skip(postPayLoadFiller); - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller, mos.position()); + // expand + 1 + mos.setLength(payLoadOffset+payLoad.length+1); + Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); // no change - // write endBytes (no resize) - mos.write(endBytes); - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.position()); + // expand up-to very end, ahead of write - position to endBytes start + mos.setLength(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length); + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); // no change + mos.skip(postPayLoadFiller); + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller, mos.position()); - // Reset to payLoad, read it and verify - mis.reset(); - Assert.assertEquals(payLoadOffset, mos.position()); - Assert.assertEquals(payLoadOffset, mis.position()); - final byte[] tmp = new byte[payLoad.length]; - Assert.assertEquals(payLoad.length, mis.read(tmp)); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertEquals(payLoadOffset+payLoad.length, mis.position()); - Assert.assertArrayEquals(payLoad, tmp); + // write endBytes (no resize) + mos.write(endBytes); + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.position()); - // Shrink to end of payLoad, mark, read >= 0, reset .. redo - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.length()); - mos.setLength(payLoadOffset+payLoad.length+1); - Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - mis.mark(1); - Assert.assertTrue(mis.read()>=0); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - mis.reset(); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertTrue(mis.read()>=0); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + // Reset to payLoad, read it and verify + mis.reset(); + Assert.assertEquals(payLoadOffset, mos.position()); + Assert.assertEquals(payLoadOffset, mis.position()); + final byte[] tmp = new byte[payLoad.length]; + Assert.assertEquals(payLoad.length, mis.read(tmp)); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertEquals(payLoadOffset+payLoad.length, mis.position()); + Assert.assertArrayEquals(payLoad, tmp); - // Shrink -1, read EOS - mos.setLength(payLoadOffset+payLoad.length); - Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertEquals(-1, mis.read()); + // Shrink to end of payLoad, mark, read >= 0, reset .. redo + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+postPayLoadFiller+endBytes.length, mos.length()); + mos.setLength(payLoadOffset+payLoad.length+1); + Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + mis.mark(1); + Assert.assertTrue(mis.read()>=0); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + mis.reset(); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertTrue(mis.read()>=0); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - // Expand + 1, mark, read >= 0, reset .. redo - mos.setLength(payLoadOffset+payLoad.length+1); - Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - mis.mark(1); - Assert.assertTrue(mis.read()>=0); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - mis.reset(); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertTrue(mis.read()>=0); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + // Shrink -1, read EOS + mos.setLength(payLoadOffset+payLoad.length); + Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertEquals(-1, mis.read()); - // Shrink -1, read EOS, write-expand, reset and verify - mos.setLength(payLoadOffset+payLoad.length); - Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertEquals(-1, mis.read()); - mos.write('Z'); // expand while writing .. - Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - mis.reset(); - Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); - Assert.assertEquals(payLoadOffset+payLoad.length, mis.position()); - Assert.assertEquals('Z', mis.read()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - Assert.assertEquals(payLoadOffset+payLoad.length+1, mis.position()); + // Expand + 1, mark, read >= 0, reset .. redo + mos.setLength(payLoadOffset+payLoad.length+1); + Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + mis.mark(1); + Assert.assertTrue(mis.read()>=0); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + mis.reset(); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertTrue(mis.read()>=0); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); - // Shrink -2, shall clear mark, test reset failure - mos.setLength(payLoadOffset+payLoad.length-1); - Assert.assertEquals(payLoadOffset+payLoad.length-1, out.length()); - Assert.assertEquals(payLoadOffset+payLoad.length-1, mos.length()); - Assert.assertEquals(payLoadOffset+payLoad.length-1, mos.position()); - try { - mis.reset(); - Assert.assertTrue(false); // shall not reach - } catch( final IOException ioe ) { - Assert.assertNotNull(ioe); - } - mis.mark(1); + // Shrink -1, read EOS, write-expand, reset and verify + mos.setLength(payLoadOffset+payLoad.length); + Assert.assertEquals(payLoadOffset+payLoad.length, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertEquals(-1, mis.read()); + mos.write('Z'); // expand while writing .. + Assert.assertEquals(payLoadOffset+payLoad.length+1, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + mis.reset(); + Assert.assertEquals(payLoadOffset+payLoad.length, mos.position()); + Assert.assertEquals(payLoadOffset+payLoad.length, mis.position()); + Assert.assertEquals('Z', mis.read()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mos.position()); + Assert.assertEquals(payLoadOffset+payLoad.length+1, mis.position()); - // ZERO file, test reset failure, read EOS, write-expand - mos.setLength(0); - Assert.assertEquals(0, out.length()); - Assert.assertEquals(0, mos.length()); - Assert.assertEquals(0, mos.position()); - try { - mis.reset(); - Assert.assertTrue(false); // shall not reach - } catch( final IOException ioe ) { - Assert.assertNotNull(ioe); - } - Assert.assertEquals(-1, mis.read()); - mos.write('Z'); // expand while writing .. - Assert.assertEquals(1, out.length()); - Assert.assertEquals(1, mos.length()); - Assert.assertEquals(1, mos.position()); - mis.position(0); - Assert.assertEquals(0, mos.position()); - Assert.assertEquals(0, mis.position()); - Assert.assertEquals('Z', mis.read()); + // Shrink -2, shall clear mark, test reset failure + mos.setLength(payLoadOffset+payLoad.length-1); + Assert.assertEquals(payLoadOffset+payLoad.length-1, out.length()); + Assert.assertEquals(payLoadOffset+payLoad.length-1, mos.length()); + Assert.assertEquals(payLoadOffset+payLoad.length-1, mos.position()); + try { + mis.reset(); + Assert.assertTrue(false); // shall not reach + } catch( final IOException ioe ) { + Assert.assertNotNull(ioe); + } + mis.mark(1); - mos.close(); - mis.close(); - out.close(); - file.delete(); + // ZERO file, test reset failure, read EOS, write-expand + mos.setLength(0); + Assert.assertEquals(0, out.length()); + Assert.assertEquals(0, mos.length()); + Assert.assertEquals(0, mos.position()); + try { + mis.reset(); + Assert.assertTrue(false); // shall not reach + } catch( final IOException ioe ) { + Assert.assertNotNull(ioe); + } + Assert.assertEquals(-1, mis.read()); + mos.write('Z'); // expand while writing .. + Assert.assertEquals(1, out.length()); + Assert.assertEquals(1, mos.length()); + Assert.assertEquals(1, mos.position()); + mis.position(0); + Assert.assertEquals(0, mos.position()); + Assert.assertEquals(0, mis.position()); + Assert.assertEquals('Z', mis.read()); + } finally { + mos.close(); + mis.close(); + out.close(); + } + } finally { + file.delete(); + } } @Test public void test00() throws IOException { final int sliceShift = 13; // 8192 bytes per slice - testImpl("./test01.bin", "123456789AB".getBytes(), 0L, 0L, "EOF".getBytes(), sliceShift); + testImpl(getSimpleTestName(".")+".bin", "123456789AB".getBytes(), 0L, 0L, "EOF".getBytes(), sliceShift); } @Test public void test01() throws IOException { final int sliceShift = 13; // 8192 bytes per slice - testImpl("./test01.bin", "123456789AB".getBytes(), 9000L, 100L, "EOF".getBytes(), sliceShift); + testImpl(getSimpleTestName(".")+".bin", "123456789AB".getBytes(), 9000L, 100L, "EOF".getBytes(), sliceShift); } @Test public void test02() throws IOException { final int sliceShift = 13; // 8192 bytes per slice - testImpl("./test01.bin", "123456789AB".getBytes(), 8189L, 9001L, "EOF".getBytes(), sliceShift); + testImpl(getSimpleTestName(".")+".bin", "123456789AB".getBytes(), 8189L, 9001L, "EOF".getBytes(), sliceShift); } @Test public void test03() throws IOException { final int sliceShift = 13; // 8192 bytes per slice - testImpl("./test01.bin", "123456789AB".getBytes(), 58189L, 109001L, "EOF".getBytes(), sliceShift); + testImpl(getSimpleTestName(".")+".bin", "123456789AB".getBytes(), 58189L, 109001L, "EOF".getBytes(), sliceShift); } @Test @@ -235,7 +251,7 @@ public class TestByteBufferOutputStream extends JunitTracer { for(int i=0; i=0 && orig[i]==bb.get(i); i--) ; + Assert.assertTrue("Bytes not equal orig vs array", 0>i); } finally { - IOUtil.close(bis, false); + file2.delete(); } - Assert.assertEquals("Byte number not equal orig vs buffer", orig.length, bb.limit()); - int i; - for(i=tsz-1; i>=0 && orig[i]==bb.get(i); i--) ; - Assert.assertTrue("Bytes not equal orig vs array", 0>i); } public static void main(final String args[]) throws IOException { -- cgit v1.2.3