HBASE-29588: New HFile Design For Multi-Tenant Support (cherry-pick of PR #7296 onto branch-2)

hbase-common/src/main/resources/hbase-default.xml

@@ -1089,10 +1089,12 @@ possible configurations would overwhelm and obscure the important.
   </property>
   <property>
       <name>hfile.format.version</name>
-      <value>3</value>
+      <value>4</value>
       <description>The HFile format version to use for new files.
       Version 3 adds support for tags in hfiles (See
       https://hbase.apache.org/docs/security/data-access#tags).
+      Version 4 introduces the multi-tenant HFile layout while remaining backward compatible
+      with older readers.
       Also see the configuration 'hbase.replication.rpc.codec'.
       </description>
   </property>

hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java

@@ -56,7 +56,7 @@ public class IntegrationTestIngestWithACL extends IntegrationTestIngest {
   public void setUpCluster() throws Exception {
     util = getTestingUtil(null);
     Configuration conf = util.getConfiguration();
-    conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
+    conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
     conf.set("hbase.coprocessor.master.classes", AccessController.class.getName());
     conf.set("hbase.coprocessor.region.classes", AccessController.class.getName());
     conf.setBoolean("hbase.security.access.early_out", false);

hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java

@@ -47,7 +47,7 @@ public void setUpCluster() throws Exception {
     Configuration conf = util.getConfiguration();
     if (!util.isDistributedCluster()) {
       // Inject required configuration if we are not running in distributed mode
-      conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
+      conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
       conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
       conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
       conf.setBoolean(HConstants.ENABLE_WAL_ENCRYPTION, true);

hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithTags.java

@@ -36,7 +36,7 @@ public class IntegrationTestIngestWithTags extends IntegrationTestIngest {
 
   @Override
   public void setUpCluster() throws Exception {
-    getTestingUtil(conf).getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3);
+    getTestingUtil(conf).getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
     super.setUpCluster();
   }
 

hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java

@@ -209,7 +209,7 @@ protected void processOptions(CommandLine cmd) {
 
   private static void initConf(Configuration conf) {
 
-    conf.setInt("hfile.format.version", 3);
+    conf.setInt("hfile.format.version", 4);
     conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 0);
     conf.setInt("hbase.client.retries.number", 100);
     conf.setInt("hbase.hregion.max.filesize", 200000000);

hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java

@@ -88,7 +88,7 @@ public static void provisionCluster() throws Exception {
     conf = util.getConfiguration();
     // We don't check persistence in HFiles in this test, but if we ever do we will
     // need this where the default hfile version is not 3 (i.e. 0.98)
-    conf.setInt("hfile.format.version", 3);
+    conf.setInt("hfile.format.version", 4);
     conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
     util.startMiniCluster();
   }

hbase-protocol-shaded/src/main/protobuf/HFile.proto

@@ -51,4 +51,10 @@ message FileTrailerProto {
   optional string comparator_class_name = 11;
   optional uint32 compression_codec = 12;
   optional bytes encryption_key = 13;
+  optional bool multi_tenant = 14;
+  optional int32 tenant_prefix_length = 15;
+  optional uint64 section_index_offset = 16;
+  optional string key_namespace = 17;
+  optional string kek_metadata = 18;
+  optional uint64 kek_checksum = 19;
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java

@@ -28,7 +28,11 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.PrivateCellUtil;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.BlockWithScanInfo;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex;
 import org.apache.hadoop.hbase.io.hfile.HFileInfo;
 import org.apache.hadoop.hbase.io.hfile.HFileReaderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
@@ -370,31 +374,97 @@ public long getFilterEntries() {
   public void close(boolean evictOnClose) throws IOException {
     if (closed.compareAndSet(false, true)) {
       if (evictOnClose) {
-        final HFileReaderImpl.HFileScannerImpl s =
-          (HFileReaderImpl.HFileScannerImpl) super.getScanner(false, true, false);
         final String reference = this.reader.getHFileInfo().getHFileContext().getHFileName();
         final String referred = StoreFileInfo.getReferredToRegionAndFile(reference).getSecond();
-        s.seekTo(splitCell);
-        if (s.getCurBlock() != null) {
-          long offset = s.getCurBlock().getOffset();
-          LOG.trace("Seeking to split cell in reader: {} for file: {} top: {}, split offset: {}",
-            this, reference, top, offset);
-          ((HFileReaderImpl) reader).getCacheConf().getBlockCache().ifPresent(cache -> {
-            int numEvictedReferred = top
-              ? cache.evictBlocksRangeByHfileName(referred, offset, Long.MAX_VALUE)
-              : cache.evictBlocksRangeByHfileName(referred, 0, offset);
-            int numEvictedReference = cache.evictBlocksByHfileName(reference);
+        // For v4 multi-tenant referred files, findSplitBlockOffset() reports a section-relative
+        // offset (the data block index reader is null at the file level for multi-tenant files,
+        // so we fall back to a section scanner whose getCurBlock() offset is local to that
+        // section). Using that as a file-level eviction range mis-targets blocks. Until we have
+        // a section-aware eviction range, fall back to evicting all blocks for the referred file
+        // when this reader is over a multi-tenant container — correctness over surgical eviction.
+        boolean referredIsMultiTenant = isReferredMultiTenant();
+        ((HFileReaderImpl) reader).getCacheConf().getBlockCache().ifPresent(cache -> {
+          int numEvictedReferred;
+          if (referredIsMultiTenant) {
+            numEvictedReferred = cache.evictBlocksByHfileName(referred);
             LOG.trace(
-              "Closing reference: {}; referred file: {}; was top? {}; evicted for referred: {};"
-                + "evicted for reference: {}",
-              reference, referred, top, numEvictedReferred, numEvictedReference);
-          });
-        }
-        s.close();
+              "Multi-tenant referred file {}: doing full-file eviction (range eviction is not"
+                + " section-aware). evictedReferred={}",
+              referred, numEvictedReferred);
+          } else {
+            long splitBlockOffset = -1L;
+            try {
+              splitBlockOffset = findSplitBlockOffset();
+            } catch (IOException e) {
+              LOG.warn("Failed to determine split block offset for reference {} (top? {});"
+                + " falling back to full-file eviction", reference, top, e);
+            }
+            if (splitBlockOffset >= 0) {
+              LOG.trace("Seeking to split cell in reader: {} for file: {} top: {}, split: {}", this,
+                reference, top, splitBlockOffset);
+              numEvictedReferred = top
+                ? cache.evictBlocksRangeByHfileName(referred, splitBlockOffset, Long.MAX_VALUE)
+                : cache.evictBlocksRangeByHfileName(referred, 0, splitBlockOffset);
+            } else {
+              LOG.debug("Unable to determine split block offset for reference {} (top? {});"
+                + " falling back to full-file eviction", reference, top);
+              numEvictedReferred = cache.evictBlocksByHfileName(referred);
+            }
+          }
+          int numEvictedReference = cache.evictBlocksByHfileName(reference);
+          LOG.trace(
+            "Closing reference: {}; referred file: {}; was top? {}; evicted for referred: {};"
+              + " evicted for reference: {}",
+            reference, referred, top, numEvictedReferred, numEvictedReference);
+        });
         reader.close(false);
       } else {
         reader.close(evictOnClose);
       }
     }
   }
+
+  /**
+   * Returns true if the underlying referred reader is a v4 multi-tenant container. Used to decide
+   * whether {@link #findSplitBlockOffset()} can produce a meaningful file-level offset for
+   * block-range eviction.
+   */
+  private boolean isReferredMultiTenant() {
+    try {
+      FixedFileTrailer trailer = reader.getTrailer();
+      return trailer != null && trailer.isMultiTenant();
+    } catch (Exception e) {
+      // Defensive: if anything is off (no trailer, wrong type), treat as multi-tenant=false.
+      return false;
+    }
+  }
+
+  private long findSplitBlockOffset() throws IOException {
+    HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
+    if (indexReader != null) {
+      BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(splitCell, null,
+        false, true, false, reader.getEffectiveEncodingInCache(false), reader);
+      if (blockWithScanInfo != null) {
+        HFileBlock block = blockWithScanInfo.getHFileBlock();
+        if (block != null) {
+          try {
+            return block.getOffset();
+          } finally {
+            block.release();
+          }
+        }
+      }
+    }
+
+    try (HFileScanner scanner = super.getScanner(false, true, false)) {
+      if (scanner instanceof HFileReaderImpl.HFileScannerImpl) {
+        HFileReaderImpl.HFileScannerImpl delegate = (HFileReaderImpl.HFileScannerImpl) scanner;
+        delegate.seekTo(splitCell);
+        if (delegate.getCurBlock() != null) {
+          return delegate.getCurBlock().getOffset();
+        }
+      }
+    }
+    return -1L;
+  }
 }