diff --git a/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java b/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java index 8729ef5..93b0011 100644 --- a/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java +++ b/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java @@ -1,5 +1,6 @@ package org.ricebin.sstable.benchmark; +import java.io.File; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -28,12 +29,11 @@ public static class MyState { @Setup(Level.Trial) public void setUpTrial() throws Exception { random = new Random(); - RandomAccessFile file = new RandomAccessFile( - "benchmark/org/ricebin/sstable/benchmark/testfiles/000005.sst", "r"); - FileChannel fc = file.getChannel(); + String filename = + "benchmark/org/ricebin/sstable/benchmark/testfiles/000005.sst"; // table = Table.open(fc, ByteBufferSlice.FACTORY); - table = Table.open(BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), fc, ByteBufferSlice.FACTORY); + table = Table.open(new File(filename), BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), ByteBufferSlice.FACTORY); } @TearDown @@ -46,7 +46,7 @@ public void setUpRandomKey() throws Exception { // byte[] sampleKey = new byte[]{ // 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 51, 51, 55, 49}; int fullkeySize = 24; - Sink keySink = ByteBufferSlice.FACTORY.newFixedSizeSink(24 + 4); + Sink keySink = ByteBufferSlice.FACTORY.newFixedSizeSink(24 + 4); for (int i = 0; i < 10; i++) { keySink.putByte((byte) 48); } diff --git a/main/org/ricebin/sstable/FileChannelReadOnlyFile.java b/main/org/ricebin/sstable/FileChannelReadOnlyFile.java new file mode 100644 index 0000000..790ec09 --- /dev/null +++ b/main/org/ricebin/sstable/FileChannelReadOnlyFile.java @@ -0,0 +1,36 @@ +package org.ricebin.sstable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; + +class FileChannelReadOnlyFile implements ReadOnlyFile { + + private final FileChannel fileChannel; + + FileChannelReadOnlyFile(FileChannel fileChannel) { + this.fileChannel = fileChannel; + } + + @Override + public ByteBuffer readFully(long pos, int len) throws IOException { + ByteBuffer buf = ByteBuffer.allocate(len).order(ByteOrder.LITTLE_ENDIAN); + int bytesRead = fileChannel.read(buf, pos); + if (bytesRead != len) { + throw new IllegalStateException("unable to read all bytes"); + } + buf.flip(); + return buf; + } + + @Override + public void close() throws IOException { + fileChannel.close(); + } + + @Override + public long size() throws IOException { + return fileChannel.size(); + } +} diff --git a/main/org/ricebin/sstable/ReadOnlyFile.java b/main/org/ricebin/sstable/ReadOnlyFile.java new file mode 100644 index 0000000..91254ce --- /dev/null +++ b/main/org/ricebin/sstable/ReadOnlyFile.java @@ -0,0 +1,14 @@ +package org.ricebin.sstable; + +import java.io.IOException; +import java.nio.ByteBuffer; + +interface ReadOnlyFile { + + ByteBuffer readFully(long pos, int len) throws IOException; + + void close() throws IOException; + + long size() throws IOException; + +} diff --git a/main/org/ricebin/sstable/SliceUtils.java b/main/org/ricebin/sstable/SliceUtils.java index 9a414ba..903ff79 100644 --- a/main/org/ricebin/sstable/SliceUtils.java +++ b/main/org/ricebin/sstable/SliceUtils.java @@ -19,16 +19,6 @@ static int sharedKeySize(Slice a, Slice b) { return minLength; } - static ByteBuffer readFully(FileChannel src, long pos, int len) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(len).order(ByteOrder.LITTLE_ENDIAN); - int bytesRead = src.read(buf, pos); - if (bytesRead != len) { - throw new IllegalStateException("unable to read all bytes"); - } - buf.flip(); - return buf; - } - static ByteBuffer duplicate(ByteBuffer buf) { return buf.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN); } diff --git a/main/org/ricebin/sstable/Table.java b/main/org/ricebin/sstable/Table.java index 133f370..7afe5f2 100644 --- a/main/org/ricebin/sstable/Table.java +++ b/main/org/ricebin/sstable/Table.java @@ -2,9 +2,10 @@ import com.google.common.collect.Iterators; import com.google.common.primitives.Ints; +import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Map; @@ -14,7 +15,7 @@ public class Table { - private final FileChannel fileChannel; + private final ReadOnlyFile inputFile; private final Slice.Factory sliceFactory; private final Block blockIndex; @@ -22,18 +23,18 @@ public class Table { final FilterBlock filterBlock; Table( - FileChannel fileChannel, + ReadOnlyFile inputFile, Slice.Factory sliceFactory, Block blockIndex, FilterBlock filterBlock) { - this.fileChannel = fileChannel; + this.inputFile = inputFile; this.sliceFactory = sliceFactory; this.blockIndex = blockIndex; this.filterBlock = filterBlock; this.getBlock = blockHandle -> { try { - return readBlock(sliceFactory, fileChannel, blockHandle, s -> s); + return readBlock(sliceFactory, inputFile, blockHandle, s -> s); } catch (IOException e) { throw new RuntimeException(e); } @@ -63,7 +64,7 @@ public Slice get(Slice key) throws IOException { return null; } - PrefixBlock valueBlock = readBlock(sliceFactory, fileChannel, valueBlockHandle, s -> s); + PrefixBlock valueBlock = readBlock(sliceFactory, inputFile, valueBlockHandle, s -> s); Iterator> valueIt = valueBlock.iterator(key); if (valueIt.hasNext()) { Entry next = valueIt.next(); @@ -88,34 +89,43 @@ public Iterator> iterator() { e -> getBlock.apply(e.getValue()).iterator())); } + public static Table openWithoutFilter( + File file, + Slice.Factory sliceFactory) throws IOException { + return open(file, null, sliceFactory); + } + + public static Table open( - FileChannel fileChannel, + File file, + FilterPolicy.Reader filterPolicy, Slice.Factory sliceFactory) throws IOException { - return open(null, fileChannel, sliceFactory); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); + return open(filterPolicy, new FileChannelReadOnlyFile(randomAccessFile.getChannel()), sliceFactory); } public static Table open( FilterPolicy.Reader filterPolicy, - FileChannel fileChannel, + ReadOnlyFile inputFile, Slice.Factory sliceFactory) throws IOException { - Footer footer = readFooter(sliceFactory, fileChannel); + Footer footer = readFooter(sliceFactory, inputFile); PrefixBlock blockIndex = readBlock( sliceFactory, - fileChannel, footer.getIndex(), + inputFile, footer.getIndex(), valueSlice -> BlockHandle.decode(valueSlice.newReader())); FilterBlock filterBlock; if (filterPolicy != null) { // https://github.com/google/leveldb/blob/f57513a1d6c99636fc5b710150d0b93713af4e43/table/table.cc#L82 - filterBlock = readFilterBlock(sliceFactory, filterPolicy, fileChannel, footer.getMetaIndex()); + filterBlock = readFilterBlock(sliceFactory, filterPolicy, inputFile, footer.getMetaIndex()); } else { filterBlock = null; } return new Table( - fileChannel, + inputFile, sliceFactory, blockIndex, filterBlock); @@ -124,12 +134,12 @@ public static Table open( static FilterBlock readFilterBlock( Slice.Factory sliceFactory, FilterPolicy.Reader filterPolicy, - FileChannel fileChannel, + ReadOnlyFile file, BlockHandle metaIndexBlockHandle) throws IOException { PrefixBlock metaIndex = readBlock( sliceFactory, - fileChannel, + file, metaIndexBlockHandle, s -> s ); @@ -146,8 +156,7 @@ static FilterBlock readFilterBlock( BlockHandle filterBlockHandle = BlockHandle.decode(data.newReader()); - ByteBuffer blockBuf = SliceUtils - .readFully(fileChannel, filterBlockHandle.getOffset(), filterBlockHandle.getSize()); + ByteBuffer blockBuf = file.readFully(filterBlockHandle.getOffset(), filterBlockHandle.getSize()); Slice filterBlockData = sliceFactory.wrap(blockBuf); @@ -158,11 +167,11 @@ static FilterBlock readFilterBlock( } static PrefixBlock readBlock( - Slice.Factory sliceFactory, FileChannel fileChannel, BlockHandle blockHandle, + Slice.Factory sliceFactory, ReadOnlyFile file, BlockHandle blockHandle, Function valueDecoder) throws IOException { // read block data + trailer - ByteBuffer dataAndTrailer = SliceUtils.readFully(fileChannel, + ByteBuffer dataAndTrailer = file.readFully( blockHandle.getOffset(), blockHandle.getSize() + BlockTrailer.MAX_ENCODED_LENGTH); @@ -178,15 +187,15 @@ static PrefixBlock readBlock( return new PrefixBlock(sliceFactory, dataSlice, sliceFactory.comparator(), valueDecoder); } - static Footer readFooter(Slice.Factory sliceFactory, FileChannel fileChannel) throws IOException { - int size = Ints.checkedCast(fileChannel.size()); + static Footer readFooter(Slice.Factory sliceFactory, ReadOnlyFile file) throws IOException { + int size = Ints.checkedCast(file.size()); long footerOffset = size - Footer.MAX_ENCODED_LENGTH; Slice slice = sliceFactory.wrap( - SliceUtils.readFully(fileChannel, footerOffset, Footer.MAX_ENCODED_LENGTH)); + file.readFully(footerOffset, Footer.MAX_ENCODED_LENGTH)); return Footer.decode(slice); } public void close() throws IOException { - fileChannel.close(); + inputFile.close(); } } \ No newline at end of file diff --git a/tests/org/ricebin/sstable/CppTableCompatTest.java b/tests/org/ricebin/sstable/CppTableCompatTest.java index 390932f..01f285c 100644 --- a/tests/org/ricebin/sstable/CppTableCompatTest.java +++ b/tests/org/ricebin/sstable/CppTableCompatTest.java @@ -4,6 +4,7 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.collect.Iterators; +import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; @@ -25,11 +26,12 @@ public class CppTableCompatTest { @Test public void testWithFilter() throws Exception { - RandomAccessFile file = new RandomAccessFile( - "tests/org/ricebin/sstable/testfiles/testWithFilter.sst", "r"); - FileChannel fc = file.getChannel(); - - Table table = Table.open(BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), fc, ByteBufferSlice.FACTORY); + String fileName = + "tests/org/ricebin/sstable/testfiles/testWithFilter.sst"; + Table table = Table.open( + new File(fileName), + BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), + ByteBufferSlice.FACTORY); assertThat(getBytes(table.filterBlock.blockContent)) .isEqualTo(new byte[]{0, 8, 64, 2, 16, 0, 4, 32, 6, 0, 0, 0, 0, 9, 0, 0, 0, 11}); @@ -48,7 +50,7 @@ public void testWithFilter() throws Exception { } @Test - public void testHappy() throws IOException { + public void test_openWithoutFilter() throws IOException { Slice firstKey = newSlice( new byte[]{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 1, 1, 0, 0, 0, 0, 0, 0}); @@ -56,11 +58,8 @@ public void testHappy() throws IOException { new byte[]{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 52, 50, 1, 43, 0, 0, 0, 0, 0, 0}); - RandomAccessFile file = new RandomAccessFile( - "tests/org/ricebin/sstable/testfiles/000005.sst", "r"); - FileChannel fc = file.getChannel(); - - Table table = Table.open(fc, ByteBufferSlice.FACTORY); + String filename = "tests/org/ricebin/sstable/testfiles/000005.sst"; + Table table = Table.openWithoutFilter(new File(filename), ByteBufferSlice.FACTORY); { Iterator> it = table.iterator(); assertThat(it.hasNext()).isTrue(); diff --git a/tests/org/ricebin/sstable/TableTest.java b/tests/org/ricebin/sstable/TableTest.java index f950af0..10216be 100644 --- a/tests/org/ricebin/sstable/TableTest.java +++ b/tests/org/ricebin/sstable/TableTest.java @@ -20,7 +20,7 @@ public class TableTest { - private static final Slice.Factory SLICE_FACTORY = ByteBufferSlice.FACTORY; + private static final Slice.Factory SLICE_FACTORY = ByteBufferSlice.FACTORY; @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); @@ -56,8 +56,7 @@ private static ImmutableMap toMap(Table table) { } private Table readTable(File file) throws IOException { - RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); - return Table.open(randomAccessFile.getChannel(), SLICE_FACTORY); + return Table.openWithoutFilter(file, SLICE_FACTORY); } private File writeTable(ImmutableMap input) throws IOException {