HBASE-29588: New HFile Design For MultiTenant Support

hbase-common/src/main/resources/hbase-default.xml

@@ -1098,10 +1098,11 @@ possible configurations would overwhelm and obscure the important.
   </property>
   <property>
       <name>hfile.format.version</name>
-      <value>3</value>
+      <value>4</value>
       <description>The HFile format version to use for new files.
-      Version 3 adds support for tags in hfiles (See http://hbase.apache.org/book.html#hbase.tags).
-      Also see the configuration 'hbase.replication.rpc.codec'.
+      Version 4 is the default for HBase 4.x and introduces the multi-tenant HFile layout while
+      remaining backward compatible with older readers. Also see the configuration
+      'hbase.replication.rpc.codec'.
       </description>
   </property>
   <property>

hbase-protocol-shaded/src/main/protobuf/server/io/HFile.proto

@@ -51,7 +51,10 @@ message FileTrailerProto {
   optional string comparator_class_name = 11;
   optional uint32 compression_codec = 12;
   optional bytes encryption_key = 13;
-  optional string key_namespace = 14;
-  optional string kek_metadata = 15;
-  optional uint64 kek_checksum = 16;
+  optional bool multiTenant = 14;
+  optional int32 tenantPrefixLength = 15;
+  optional uint64 section_index_offset = 16;
+  optional string key_namespace = 17;
+  optional string kek_metadata = 18;
+  optional uint64 kek_checksum = 19;
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java

@@ -28,7 +28,10 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.PrivateCellUtil;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.BlockWithScanInfo;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex;
 import org.apache.hadoop.hbase.io.hfile.HFileInfo;
 import org.apache.hadoop.hbase.io.hfile.HFileReaderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
@@ -356,31 +359,59 @@ public long getFilterEntries() {
   public void close(boolean evictOnClose) throws IOException {
     if (closed.compareAndSet(false, true)) {
       if (evictOnClose) {
-        final HFileReaderImpl.HFileScannerImpl s =
-          (HFileReaderImpl.HFileScannerImpl) super.getScanner(false, true, false);
+        long splitBlockOffset = findSplitBlockOffset();
         final String reference = this.reader.getHFileInfo().getHFileContext().getHFileName();
         final String referred = StoreFileInfo.getReferredToRegionAndFile(reference).getSecond();
-        s.seekTo(splitCell);
-        if (s.getCurBlock() != null) {
-          long offset = s.getCurBlock().getOffset();
+        if (splitBlockOffset >= 0) {
           LOG.trace("Seeking to split cell in reader: {} for file: {} top: {}, split offset: {}",
-            this, reference, top, offset);
+            this, reference, top, splitBlockOffset);
           ((HFileReaderImpl) reader).getCacheConf().getBlockCache().ifPresent(cache -> {
             int numEvictedReferred = top
-              ? cache.evictBlocksRangeByHfileName(referred, offset, Long.MAX_VALUE)
-              : cache.evictBlocksRangeByHfileName(referred, 0, offset);
+              ? cache.evictBlocksRangeByHfileName(referred, splitBlockOffset, Long.MAX_VALUE)
+              : cache.evictBlocksRangeByHfileName(referred, 0, splitBlockOffset);
             int numEvictedReference = cache.evictBlocksByHfileName(reference);
             LOG.trace(
               "Closing reference: {}; referred file: {}; was top? {}; evicted for referred: {};"
                 + "evicted for reference: {}",
               reference, referred, top, numEvictedReferred, numEvictedReference);
           });
+        } else {
+          LOG.debug("Unable to determine split block offset for reference {} (top? {})", reference,
+            top);
         }
-        s.close();
         reader.close(false);
       } else {
         reader.close(evictOnClose);
       }
     }
   }
+
+  private long findSplitBlockOffset() throws IOException {
+    HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
+    if (indexReader != null) {
+      BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(splitCell, null,
+        false, true, false, reader.getEffectiveEncodingInCache(false), reader);
+      if (blockWithScanInfo != null) {
+        HFileBlock block = blockWithScanInfo.getHFileBlock();
+        if (block != null) {
+          try {
+            return block.getOffset();
+          } finally {
+            block.release();
+          }
+        }
+      }
+    }
+
+    try (HFileScanner scanner = super.getScanner(false, true, false)) {
+      if (scanner instanceof HFileReaderImpl.HFileScannerImpl) {
+        HFileReaderImpl.HFileScannerImpl delegate = (HFileReaderImpl.HFileScannerImpl) scanner;
+        delegate.seekTo(splitCell);
+        if (delegate.getCurBlock() != null) {
+          return delegate.getCurBlock().getOffset();
+        }
+      }
+    }
+    return -1L;
+  }
 }