readonnly input file interface

ricebin · May 21, 2021 · 12d410d · 12d410d
1 parent 2bc4c01
commit 12d410d
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 52 deletions.
diff --git a/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java b/benchmark/org/ricebin/sstable/benchmark/RandomGetBenchmark.java
@@ -1,5 +1,6 @@
 package org.ricebin.sstable.benchmark;
 
+import java.io.File;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
@@ -28,12 +29,11 @@ public static class MyState {
     @Setup(Level.Trial)
     public void setUpTrial() throws Exception {
       random = new Random();
-      RandomAccessFile file = new RandomAccessFile(
-          "benchmark/org/ricebin/sstable/benchmark/testfiles/000005.sst", "r");
-      FileChannel fc = file.getChannel();
+      String filename =
+          "benchmark/org/ricebin/sstable/benchmark/testfiles/000005.sst";
 
 //      table = Table.open(fc, ByteBufferSlice.FACTORY);
-      table = Table.open(BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), fc, ByteBufferSlice.FACTORY);
+      table = Table.open(new File(filename), BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), ByteBufferSlice.FACTORY);
     }
 
     @TearDown
@@ -46,7 +46,7 @@ public void setUpRandomKey() throws Exception {
 //      byte[] sampleKey = new byte[]{
 //          48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 51, 51, 55, 49};
       int fullkeySize = 24;
-      Sink keySink = ByteBufferSlice.FACTORY.newFixedSizeSink(24 + 4);
+      Sink<ByteBufferSlice> keySink = ByteBufferSlice.FACTORY.newFixedSizeSink(24 + 4);
       for (int i = 0; i < 10; i++) {
         keySink.putByte((byte) 48);
       }

diff --git a/main/org/ricebin/sstable/FileChannelReadOnlyFile.java b/main/org/ricebin/sstable/FileChannelReadOnlyFile.java
@@ -0,0 +1,36 @@
+package org.ricebin.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+
+class FileChannelReadOnlyFile implements ReadOnlyFile {
+
+  private final FileChannel fileChannel;
+
+  FileChannelReadOnlyFile(FileChannel fileChannel) {
+    this.fileChannel = fileChannel;
+  }
+
+  @Override
+  public ByteBuffer readFully(long pos, int len) throws IOException {
+    ByteBuffer buf = ByteBuffer.allocate(len).order(ByteOrder.LITTLE_ENDIAN);
+    int bytesRead = fileChannel.read(buf, pos);
+    if (bytesRead != len) {
+      throw new IllegalStateException("unable to read all bytes");
+    }
+    buf.flip();
+    return buf;
+  }
+
+  @Override
+  public void close() throws IOException {
+    fileChannel.close();
+  }
+
+  @Override
+  public long size() throws IOException {
+    return fileChannel.size();
+  }
+}
diff --git a/main/org/ricebin/sstable/ReadOnlyFile.java b/main/org/ricebin/sstable/ReadOnlyFile.java
@@ -0,0 +1,14 @@
+package org.ricebin.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+interface ReadOnlyFile {
+
+  ByteBuffer readFully(long pos, int len) throws IOException;
+
+  void close() throws IOException;
+
+  long size() throws IOException;
+
+}
diff --git a/main/org/ricebin/sstable/SliceUtils.java b/main/org/ricebin/sstable/SliceUtils.java
@@ -19,16 +19,6 @@ static int sharedKeySize(Slice a, Slice b) {
     return minLength;
   }
 
-  static ByteBuffer readFully(FileChannel src, long pos, int len) throws IOException {
-    ByteBuffer buf = ByteBuffer.allocate(len).order(ByteOrder.LITTLE_ENDIAN);
-    int bytesRead = src.read(buf, pos);
-    if (bytesRead != len) {
-      throw new IllegalStateException("unable to read all bytes");
-    }
-    buf.flip();
-    return buf;
-  }
-
   static ByteBuffer duplicate(ByteBuffer buf) {
     return buf.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN);
   }

diff --git a/main/org/ricebin/sstable/Table.java b/main/org/ricebin/sstable/Table.java
@@ -2,9 +2,10 @@
 
 import com.google.common.collect.Iterators;
 import com.google.common.primitives.Ints;
+import java.io.File;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.Iterator;
 import java.util.Map;
@@ -14,26 +15,26 @@
 
 public class Table {
 
-  private final FileChannel fileChannel;
+  private final ReadOnlyFile inputFile;
   private final Slice.Factory sliceFactory;
 
   private final Block<BlockHandle> blockIndex;
   private final Function<BlockHandle, Block<Slice>> getBlock;
   final FilterBlock filterBlock;
 
   Table(
-      FileChannel fileChannel,
+      ReadOnlyFile inputFile,
       Slice.Factory sliceFactory,
       Block<BlockHandle> blockIndex,
       FilterBlock filterBlock) {
-    this.fileChannel = fileChannel;
+    this.inputFile = inputFile;
     this.sliceFactory = sliceFactory;
     this.blockIndex = blockIndex;
     this.filterBlock = filterBlock;
     this.getBlock =
         blockHandle -> {
           try {
-            return readBlock(sliceFactory, fileChannel, blockHandle, s -> s);
+            return readBlock(sliceFactory, inputFile, blockHandle, s -> s);
           } catch (IOException e) {
             throw new RuntimeException(e);
           }
@@ -63,7 +64,7 @@ public Slice get(Slice key) throws IOException {
       return null;
     }
 
-    PrefixBlock<Slice> valueBlock = readBlock(sliceFactory, fileChannel, valueBlockHandle, s -> s);
+    PrefixBlock<Slice> valueBlock = readBlock(sliceFactory, inputFile, valueBlockHandle, s -> s);
     Iterator<Entry<Slice, Slice>> valueIt = valueBlock.iterator(key);
     if (valueIt.hasNext()) {
       Entry<Slice, Slice> next = valueIt.next();
@@ -88,34 +89,43 @@ public Iterator<Map.Entry<Slice, Slice>> iterator() {
             e -> getBlock.apply(e.getValue()).iterator()));
   }
 
+  public static Table openWithoutFilter(
+      File file,
+      Slice.Factory sliceFactory) throws IOException {
+    return open(file, null, sliceFactory);
+  }
+
+
   public static Table open(
-      FileChannel fileChannel,
+      File file,
+      FilterPolicy.Reader filterPolicy,
       Slice.Factory sliceFactory) throws IOException {
-    return open(null, fileChannel, sliceFactory);
+    RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
+    return open(filterPolicy, new FileChannelReadOnlyFile(randomAccessFile.getChannel()), sliceFactory);
   }
 
   public static Table open(
       FilterPolicy.Reader filterPolicy,
-      FileChannel fileChannel,
+      ReadOnlyFile inputFile,
       Slice.Factory sliceFactory) throws IOException {
-    Footer footer = readFooter(sliceFactory, fileChannel);
+    Footer footer = readFooter(sliceFactory, inputFile);
 
     PrefixBlock<BlockHandle> blockIndex = readBlock(
         sliceFactory,
-        fileChannel, footer.getIndex(),
+        inputFile, footer.getIndex(),
         valueSlice -> BlockHandle.decode(valueSlice.newReader()));
 
     FilterBlock filterBlock;
     if (filterPolicy != null) {
 
       // https://github.com/google/leveldb/blob/f57513a1d6c99636fc5b710150d0b93713af4e43/table/table.cc#L82
-      filterBlock = readFilterBlock(sliceFactory, filterPolicy, fileChannel, footer.getMetaIndex());
+      filterBlock = readFilterBlock(sliceFactory, filterPolicy, inputFile, footer.getMetaIndex());
     } else {
       filterBlock = null;
     }
 
     return new Table(
-        fileChannel,
+        inputFile,
         sliceFactory,
         blockIndex,
         filterBlock);
@@ -124,12 +134,12 @@ public static Table open(
   static FilterBlock readFilterBlock(
       Slice.Factory sliceFactory,
       FilterPolicy.Reader filterPolicy,
-      FileChannel fileChannel,
+      ReadOnlyFile file,
       BlockHandle metaIndexBlockHandle)
       throws IOException {
     PrefixBlock<Slice> metaIndex = readBlock(
         sliceFactory,
-        fileChannel,
+        file,
         metaIndexBlockHandle,
         s -> s
     );
@@ -146,8 +156,7 @@ static FilterBlock readFilterBlock(
 
         BlockHandle filterBlockHandle = BlockHandle.decode(data.newReader());
 
-        ByteBuffer blockBuf = SliceUtils
-            .readFully(fileChannel, filterBlockHandle.getOffset(), filterBlockHandle.getSize());
+        ByteBuffer blockBuf = file.readFully(filterBlockHandle.getOffset(), filterBlockHandle.getSize());
 
         Slice filterBlockData = sliceFactory.wrap(blockBuf);
 
@@ -158,11 +167,11 @@ static FilterBlock readFilterBlock(
   }
 
   static <V> PrefixBlock<V> readBlock(
-      Slice.Factory sliceFactory, FileChannel fileChannel, BlockHandle blockHandle,
+      Slice.Factory sliceFactory, ReadOnlyFile file, BlockHandle blockHandle,
       Function<Slice, V> valueDecoder) throws IOException {
 
     // read block data + trailer
-    ByteBuffer dataAndTrailer = SliceUtils.readFully(fileChannel,
+    ByteBuffer dataAndTrailer = file.readFully(
         blockHandle.getOffset(),
         blockHandle.getSize() + BlockTrailer.MAX_ENCODED_LENGTH);
 
@@ -178,15 +187,15 @@ static <V> PrefixBlock<V> readBlock(
     return new PrefixBlock<V>(sliceFactory, dataSlice, sliceFactory.comparator(), valueDecoder);
   }
 
-  static Footer readFooter(Slice.Factory sliceFactory, FileChannel fileChannel) throws IOException {
-    int size = Ints.checkedCast(fileChannel.size());
+  static Footer readFooter(Slice.Factory sliceFactory, ReadOnlyFile file) throws IOException {
+    int size = Ints.checkedCast(file.size());
     long footerOffset = size - Footer.MAX_ENCODED_LENGTH;
     Slice slice = sliceFactory.wrap(
-        SliceUtils.readFully(fileChannel, footerOffset, Footer.MAX_ENCODED_LENGTH));
+        file.readFully(footerOffset, Footer.MAX_ENCODED_LENGTH));
     return Footer.decode(slice);
   }
 
   public void close() throws IOException {
-    fileChannel.close();
+    inputFile.close();
   }
 }
diff --git a/tests/org/ricebin/sstable/CppTableCompatTest.java b/tests/org/ricebin/sstable/CppTableCompatTest.java
@@ -4,6 +4,7 @@
 import static com.google.common.truth.Truth.assertThat;
 
 import com.google.common.collect.Iterators;
+import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.channels.FileChannel;
@@ -25,11 +26,12 @@ public class CppTableCompatTest {
 
   @Test
   public void testWithFilter() throws Exception {
-    RandomAccessFile file = new RandomAccessFile(
-        "tests/org/ricebin/sstable/testfiles/testWithFilter.sst", "r");
-    FileChannel fc = file.getChannel();
-
-    Table table = Table.open(BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(), fc, ByteBufferSlice.FACTORY);
+    String fileName =
+        "tests/org/ricebin/sstable/testfiles/testWithFilter.sst";
+    Table table = Table.open(
+        new File(fileName),
+        BloomFilterPolicy.LEVELDB_BUILTIN_BLOOM_FILTER2.getReader(),
+        ByteBufferSlice.FACTORY);
 
     assertThat(getBytes(table.filterBlock.blockContent))
         .isEqualTo(new byte[]{0, 8, 64, 2, 16, 0, 4, 32, 6, 0, 0, 0, 0, 9, 0, 0, 0, 11});
@@ -48,19 +50,16 @@ public void testWithFilter() throws Exception {
   }
 
   @Test
-  public void testHappy() throws IOException {
+  public void test_openWithoutFilter() throws IOException {
     Slice firstKey = newSlice(
         new byte[]{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 1, 1, 0, 0, 0, 0,
             0, 0});
     Slice midKey = newSlice(
         new byte[]{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 52, 50, 1, 43, 0, 0, 0,
             0, 0, 0});
 
-    RandomAccessFile file = new RandomAccessFile(
-        "tests/org/ricebin/sstable/testfiles/000005.sst", "r");
-    FileChannel fc = file.getChannel();
-
-    Table table = Table.open(fc, ByteBufferSlice.FACTORY);
+    String filename = "tests/org/ricebin/sstable/testfiles/000005.sst";
+    Table table = Table.openWithoutFilter(new File(filename), ByteBufferSlice.FACTORY);
     {
       Iterator<Entry<Slice, Slice>> it = table.iterator();
       assertThat(it.hasNext()).isTrue();

diff --git a/tests/org/ricebin/sstable/TableTest.java b/tests/org/ricebin/sstable/TableTest.java
@@ -20,7 +20,7 @@
 
 public class TableTest {
 
-  private static final Slice.Factory SLICE_FACTORY = ByteBufferSlice.FACTORY;
+  private static final Slice.Factory<ByteBufferSlice> SLICE_FACTORY = ByteBufferSlice.FACTORY;
 
   @Rule
   public final TemporaryFolder tempDir = new TemporaryFolder();
@@ -56,8 +56,7 @@ private static ImmutableMap<String, String> toMap(Table table) {
   }
 
   private Table readTable(File file) throws IOException {
-    RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
-    return Table.open(randomAccessFile.getChannel(), SLICE_FACTORY);
+    return Table.openWithoutFilter(file, SLICE_FACTORY);
   }
 
   private File writeTable(ImmutableMap<String, String> input) throws IOException {