diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto index 001cb3ea233c..985c693b9c6f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto @@ -25,9 +25,11 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; +import "server/io/FS.proto"; message StoreFileEntry { required string name = 1; required uint64 size = 2; + optional Reference reference = 3; } message StoreFileList { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index bd5fac1c3c45..b31ebfe578d6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -81,7 +81,7 @@ public class HFileLink extends FileLink { * The pattern should be used for hfile and reference links that can be found in * /hbase/table/region/family/ */ - private static final Pattern REF_OR_HFILE_LINK_PATTERN = + public static final Pattern REF_OR_HFILE_LINK_PATTERN = Pattern.compile(String.format("^(?:(%s)(?:=))?(%s)=(%s)-(.+)$", TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, RegionInfoBuilder.ENCODED_REGION_NAME_REGEX)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java index 5388a1105c33..ab305d02f0f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java @@ -56,7 +56,7 @@ public class Reference { * For split HStoreFiles, it specifies if the file covers the lower half or the upper half of the * key range */ - static enum Range { + public static enum Range { /** HStoreFile contains upper half of key range */ top, /** HStoreFile contains lower half of key range */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index c370fed9d9c0..8cc447c52246 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -605,7 +605,7 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); - List mergedFiles = new ArrayList<>(); + List mergedFiles = new ArrayList(); HRegionFileSystem mergeRegionFs = HRegionFileSystem .createRegionOnFileSystem(env.getMasterConfiguration(), fs, tableDir, mergedRegion); @@ -622,11 +622,11 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException .setState(State.MERGING_NEW); } - private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, + private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable()); - List mergedFiles = new ArrayList<>(); + List mergedFiles = new ArrayList(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); StoreFileTracker tracker = @@ -643,7 +643,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg // is running in a regionserver's Store context, or we might not be able // to read the hfiles. storeFileInfo.setConf(storeConfiguration); - Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, + StoreFileInfo refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker); mergedFiles.add(refFile); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 3d3d3d18de23..a045721146b7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -660,20 +660,46 @@ public void createDaughterRegions(final MasterProcedureEnv env) throws IOExcepti HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); regionFs.createSplitsDir(daughterOneRI, daughterTwoRI); + Pair, List> expectedReferences = + splitStoreFiles(env, regionFs); + final ExecutorService threadPool = Executors.newFixedThreadPool(2, + new ThreadFactoryBuilder().setNameFormat("RegionCommitter-pool-%d").setDaemon(true) + .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build()); + threadPool.submit(new Callable() { + @Override + public Path call() throws IOException { + return regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env); + } + }); + threadPool.submit(new Callable() { + @Override + public Path call() throws IOException { + return regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env); + } + }); + // Shutdown the pool + threadPool.shutdown(); - Pair, List> expectedReferences = splitStoreFiles(env, regionFs); - - assertSplitResultFilesCount(fs, expectedReferences.getFirst().size(), - regionFs.getSplitsDir(daughterOneRI)); - regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env); - assertSplitResultFilesCount(fs, expectedReferences.getFirst().size(), - new Path(tabledir, daughterOneRI.getEncodedName())); - - assertSplitResultFilesCount(fs, expectedReferences.getSecond().size(), - regionFs.getSplitsDir(daughterTwoRI)); - regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env); - assertSplitResultFilesCount(fs, expectedReferences.getSecond().size(), - new Path(tabledir, daughterTwoRI.getEncodedName())); + Configuration conf = env.getMasterConfiguration(); + // Wait for all the tasks to finish. + // When splits ran on the RegionServer, how-long-to-wait-configuration was named + // hbase.regionserver.fileSplitTimeout. If set, use its value. + long fileSplitTimeout = conf.getLong("hbase.master.fileSplitTimeout", + conf.getLong("hbase.regionserver.fileSplitTimeout", 600000)); + try { + boolean stillRunning = !threadPool.awaitTermination(fileSplitTimeout, TimeUnit.MILLISECONDS); + if (stillRunning) { + threadPool.shutdownNow(); + // wait for the thread to shutdown completely. + while (!threadPool.isTerminated()) { + Thread.sleep(50); + } + throw new IOException( + "Took too long to split the" + " files and create the references, aborting split"); + } + } catch (InterruptedException e) { + throw (InterruptedIOException) new InterruptedIOException().initCause(e); + } } private void deleteDaughterRegions(final MasterProcedureEnv env) throws IOException { @@ -689,8 +715,8 @@ private void deleteDaughterRegions(final MasterProcedureEnv env) throws IOExcept * Create Split directory * @param env MasterProcedureEnv */ - private Pair, List> splitStoreFiles(final MasterProcedureEnv env, - final HRegionFileSystem regionFs) throws IOException { + private Pair, List> splitStoreFiles( + final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException { final Configuration conf = env.getMasterConfiguration(); TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); // The following code sets up a thread pool executor with as many slots as @@ -745,7 +771,8 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en final ExecutorService threadPool = Executors.newFixedThreadPool(maxThreads, new ThreadFactoryBuilder().setNameFormat("StoreFileSplitter-pool-%d").setDaemon(true) .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build()); - final List>> futures = new ArrayList>>(nbFiles); + final List>> futures = + new ArrayList>>(nbFiles); // Split each store file. for (Map.Entry> e : files.entrySet()) { @@ -792,12 +819,12 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en throw (InterruptedIOException) new InterruptedIOException().initCause(e); } - List daughterA = new ArrayList<>(); - List daughterB = new ArrayList<>(); + List daughterA = new ArrayList<>(); + List daughterB = new ArrayList<>(); // Look for any exception - for (Future> future : futures) { + for (Future> future : futures) { try { - Pair p = future.get(); + Pair p = future.get(); if (p.getFirst() != null) { daughterA.add(p.getFirst()); } @@ -819,6 +846,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en return new Pair<>(daughterA, daughterB); } + // TODO: update assert to do SFT.load instead of FileSystem listing private void assertSplitResultFilesCount(final FileSystem fs, final int expectedSplitResultFileCount, Path dir) throws IOException { if (expectedSplitResultFileCount != 0) { @@ -830,8 +858,8 @@ private void assertSplitResultFilesCount(final FileSystem fs, } } - private Pair splitStoreFile(HRegionFileSystem regionFs, TableDescriptor htd, - ColumnFamilyDescriptor hcd, HStoreFile sf) throws IOException { + private Pair splitStoreFile(HRegionFileSystem regionFs, + TableDescriptor htd, ColumnFamilyDescriptor hcd, HStoreFile sf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); @@ -847,22 +875,22 @@ private Pair splitStoreFile(HRegionFileSystem regionFs, TableDescrip StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd, hcd, HRegionFileSystem.create(regionFs.getFileSystem().getConf(), regionFs.getFileSystem(), regionFs.getTableDir(), daughterTwoRI)); - final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, - false, splitPolicy, daughterOneSft); - final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, - true, splitPolicy, daughterTwoSft); + final StoreFileInfo sfiFirst = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, + splitRow, false, splitPolicy, daughterOneSft); + final StoreFileInfo sfiSecond = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, + splitRow, true, splitPolicy, daughterTwoSft); if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); } - return new Pair(path_first, path_second); + return new Pair(sfiFirst, sfiSecond); } /** * Utility class used to do the file splitting / reference writing in parallel instead of * sequentially. */ - private class StoreFileSplitter implements Callable> { + private class StoreFileSplitter implements Callable> { private final HRegionFileSystem regionFs; private final ColumnFamilyDescriptor hcd; private final HStoreFile sf; @@ -883,7 +911,7 @@ public StoreFileSplitter(HRegionFileSystem regionFs, TableDescriptor htd, } @Override - public Pair call() throws IOException { + public Pair call() throws IOException { return splitStoreFile(regionFs, htd, hcd, sf); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index f828fe33cea5..54f3bfcc22a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -102,6 +102,7 @@ public class HMobStore extends HStore { // table, we need to find the original mob files by this table name. For details please see // cloning snapshot for mob files. private final byte[] refCellTags; + private StoreFileTracker mobStoreSFT = null; public HMobStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam, boolean warmup) throws IOException { @@ -273,7 +274,36 @@ public void commitFile(final Path sourceFile, Path targetPath) throws IOExceptio if (!getFileSystem().rename(sourceFile, dstPath)) { throw new IOException("Failed rename of " + sourceFile + " to " + dstPath); } - } + + // LOG.info("MOB SFT Debug - Destination path: {}", dstPath); + // LOG.info("MOB SFT Debug - Tracker class: {}", mobStoreSFT.getClass().getName()); + // mobStoreSFT.add(Collections.singleton(StoreFileInfo.createStoreFileInfoForHFile(conf, + // getFileSystem(), dstPath, true))); + } + + // private StoreFileTracker getMobStoreSFT() throws IOException { + // FileSystem fs = getFileSystem(); + // HRegionFileSystem regionFS = (fs.exists(MobUtils.getMobRegionPath(conf,getTableName())) + // ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, MobUtils.getMobTableDir(conf, + // getTableName()), MobUtils.getMobRegionInfo(getTableName()), + // false) + // : HRegionFileSystem.createRegionOnFileSystem(conf, fs, MobUtils.getMobTableDir(conf, + // getTableName()), MobUtils.getMobRegionInfo(getTableName()))); + // StoreContext storeContext = + // StoreContext.getBuilder().withColumnFamilyDescriptor(getStoreContext().getFamily()) + // .withRegionFileSystem(regionFS) + // .withFamilyStoreDirectoryPath(MobUtils.getMobFamilyPath(conf, getTableName(), + // getStoreContext().getFamily().getNameAsString())) + // .withColumnFamilyDescriptor(getStoreContext().getFamily()) + // .build(); + // StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true,storeContext); + // // *** ADD DEBUG LOGGING *** + // + // LOG.info("MOB SFT Debug - Store context family dir: {}", + // storeContext.getFamilyStoreDirectoryPath()); + // LOG.info("MOB SFT Debug - Region info: {}", storeContext.getRegionInfo()); + // return sft; + // } /** * Validates a mob file by opening and closing it. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 7f766c558409..b4bf2205ab94 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -496,7 +496,7 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * in the filesystem. * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} */ - public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, + public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, MasterProcedureEnv env) throws IOException { Path regionDir = this.getSplitsDir(regionInfo); if (fs.exists(regionDir)) { @@ -511,21 +511,43 @@ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegi return regionDir; } - private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProcedureEnv env, - HRegionFileSystem regionFs) throws IOException { + private void insertRegionFilesIntoStoreTracker(List allFiles, + MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException { TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors().get(regionInfo.getTable()); // we need to map trackers per store Map trackerMap = new HashMap<>(); // we need to map store files per store Map> fileInfoMap = new HashMap<>(); - for (Path file : allFiles) { + for (StoreFileInfo sfi : allFiles) { + Path file = sfi.getPath(); String familyName = file.getParent().getName(); trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); - infos.add(trackerMap.get(familyName).getStoreFileInfo(file, true)); + infos.add(sfi); + } + for (Map.Entry entry : trackerMap.entrySet()) { + entry.getValue().add(fileInfoMap.get(entry.getKey())); + } + } + + private void insertRegionfilePathsIntoStoreTracker(List allFiles, + MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException { + TableDescriptor tblDesc = + env.getMasterServices().getTableDescriptors().get(regionInfo.getTable()); + // we need to map trackers per store + Map trackerMap = new HashMap<>(); + // we need to map store files per store + Map> fileInfoMap = new HashMap<>(); + for (StoreFileInfo file : allFiles) { + String familyName = file.getPath().getParent().getName(); + trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, + tblDesc.getColumnFamily(familyName.getBytes()), regionFs)); + fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); + List infos = fileInfoMap.get(familyName); + infos.add(file); } for (Map.Entry entry : trackerMap.entrySet()) { entry.getValue().add(fileInfoMap.get(entry.getKey())); @@ -568,8 +590,9 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * have a reference to a Region. * @return Path to created reference. */ - public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, - boolean top, RegionSplitPolicy splitPolicy, StoreFileTracker tracker) throws IOException { + public StoreFileInfo splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, + byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy, StoreFileTracker tracker) + throws IOException { Path splitDir = new Path(getSplitsDir(hri), familyName); // Add the referred-to regions name as a dot separated suffix. // See REF_NAME_REGEX regex above. The referred-to regions name is @@ -581,7 +604,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName); if (fs.exists(p)) { LOG.warn("Found an already existing split file for {}. Assuming this is a recovery.", p); - return p; + return tracker.getStoreFileInfo(fs.getFileStatus(p), p, true); } boolean createLinkFile = false; if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) { @@ -639,12 +662,12 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte hfileName = m.group(4); } // must create back reference here - tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true); + HFileLink hFileLink = tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true); Path path = new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName)); LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName() + ", parent: " + regionInfoForFs.getEncodedName()); - return path; + return new StoreFileInfo(conf, fs, path, hFileLink); } catch (IOException e) { // if create HFileLink file failed, then just skip the error and create Reference file LOG.error("Create link file for " + hfileName + " for child " + hri.getEncodedName() @@ -655,7 +678,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte Reference r = top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow); tracker.createReference(r, p); - return p; + return new StoreFileInfo(conf, fs, p, r); } // =========================================================================== @@ -696,7 +719,7 @@ static boolean mkdirs(FileSystem fs, Configuration conf, Path dir) throws IOExce * @return Path to created reference. * @throws IOException if the merge write fails. */ - public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f, + public StoreFileInfo mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f, StoreFileTracker tracker) throws IOException { Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName); // A whole reference to the store file. @@ -710,13 +733,14 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi // suffix and into the new region location (under same family). Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName); tracker.createReference(r, p); - return p; + StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, p, r); + return storeFileInfo; } /** * Commit a merged region, making it ready for use. */ - public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env) + public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env) throws IOException { Path regionDir = getMergesDir(regionInfoForFs); if (regionDir != null && fs.exists(regionDir)) { @@ -724,7 +748,7 @@ public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); - insertRegionFilesIntoStoreTracker(allMergedFiles, env, this); + insertRegionfilePathsIntoStoreTracker(allMergedFiles, env, this); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 98299c47302c..84db33b6e271 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -919,6 +919,8 @@ private boolean completeFlush(final List sfs, long snapshotId) throw HStoreFile.increaseStoreFilesRefeCount(sfs); }); // notify to be called here - only in case of flushes + System.out.println("region name here - " + getStoreContext().getRegionInfo().getEncodedName()); + System.out.println("Added these store files to engine " + sfs); try { notifyChangedReadersObservers(sfs); } finally { @@ -1389,10 +1391,15 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick // some of the input files might already be deleted List inputStoreFiles = new ArrayList<>(compactionInputs.size()); + System.out.println("print getstorefiles - " + this.getStorefiles()); for (HStoreFile sf : this.getStorefiles()) { if (inputFiles.contains(sf.getPath().getName())) { inputStoreFiles.add(sf); } + // else if(!isPrimaryReplicaStore() && sf.getFileInfo().isLink() && + // inputFiles.contains(HFileLink.getReferencedHFileName(sf.getPath().getName()))) { + // inputStoreFiles.add(sf); + // } } // check whether we need to pick up the new files @@ -1403,7 +1410,8 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick compactionOutputs.remove(sf.getPath().getName()); } for (String compactionOutput : compactionOutputs) { - StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); + StoreFileTracker sft = + StoreFileTrackerFactory.create(conf, isPrimaryReplicaStore(), storeContext); StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); @@ -1414,6 +1422,8 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) { LOG.info("Replaying compaction marker, replacing input files: " + inputStoreFiles + " with output files : " + outputStoreFiles); + LOG.info( + "actual input files: " + inputFiles + " with output files of input : " + outputStoreFiles); this.replaceStoreFiles(inputStoreFiles, outputStoreFiles, false); this.refreshStoreSizeAndTotalBytes(); } @@ -2046,9 +2056,13 @@ public List getCommittedFiles() { public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) throws IOException { List storeFiles = new ArrayList<>(fileNames.size()); + System.out + .println("region name here - " + getStoreContext().getRegionInfo().getEncodedName()); + System.out.println("Actual filenames I got for replay of flush " + fileNames); for (String file : fileNames) { // open the file as a store file (hfile link, etc) - StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); + StoreFileTracker sft = + StoreFileTrackerFactory.create(conf, isPrimaryReplicaStore(), storeContext); StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 30cf5e2a92fa..b2e885530758 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -511,6 +511,10 @@ public void addStoreFiles(Collection storeFiles, public void replaceStoreFiles(Collection compactedFiles, Collection newFiles, IOExceptionRunnable walMarkerWriter, Runnable actionUnderLock) throws IOException { + // System.out.println(ctx.getRegionInfo().getEncodedName()); + // System.out.println("Replacing files in SFT for "); + // System.out.println("old files - " + compactedFiles); + // System.out.println("new files - " + newFiles); storeFileTracker.replace(StoreUtils.toStoreFileInfo(compactedFiles), StoreUtils.toStoreFileInfo(newFiles)); walMarkerWriter.run(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index cd748bb353a6..84ed7ea0ed54 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -171,6 +171,17 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt this(conf, fs, fileStatus, null, link); } + /** + * Create a Store File Info from an HFileLink + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + */ + public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initiaPath, + final HFileLink link) { + this(conf, fs, initiaPath, null, link); + } + /** * Create a Store File Info from an HFileLink * @param conf The {@link Configuration} to use @@ -183,6 +194,18 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt this(conf, fs, fileStatus, reference, null); } + /** + * Create a Store File Info from an HFileLink + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + * @param reference The reference instance + */ + public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath, + final Reference reference) { + this(conf, fs, initialPath, reference, null); + } + /** * Create a Store File Info from an HFileLink and a Reference * @param conf The {@link Configuration} to use @@ -204,6 +227,26 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD); } + /** + * Create a Store File Info from an HFileLink and a Reference + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + * @param reference The reference instance + * @param link The link instance + */ + public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path, + final Reference reference, final HFileLink link) { + this.fs = fs; + this.conf = conf; + this.primaryReplica = false; + this.initialPath = path; + this.reference = reference; + this.link = link; + this.noReadahead = + this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD); + } + /** * Create a Store File Info from an HFileLink and a Reference * @param conf The {@link Configuration} to use @@ -603,12 +646,17 @@ public static Pair getReferredToRegionAndFile(final String refer * @return true if the file could be a valid store file, false otherwise */ public static boolean validateStoreFileName(final String fileName) { - if (HFileLink.isHFileLink(fileName) || isReference(fileName)) { + if (HFileLink.isHFileLink(fileName) || isReference(fileName) || isMobFileLink(fileName)) { return true; } return !fileName.contains("-"); } + private static boolean isMobFileLink(String fileName) { + Matcher m = HFileLink.REF_OR_HFILE_LINK_PATTERN.matcher(fileName); + return m.matches(); + } + /** * Return if the specified file is a valid store file or not. * @param fileStatus The {@link FileStatus} of the file @@ -624,7 +672,7 @@ public static boolean isValid(final FileStatus fileStatus) throws IOException { // Check for empty hfile. Should never be the case but can happen // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646 // NOTE: that the HFileLink is just a name, so it's an empty file. - if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) { + if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0 && !isMobFileLink(p.getName())) { LOG.warn("Skipping {} because it is empty. HBASE-646 DATA LOSS?", p); return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java index c9ee019e9afe..4e4118fb4d7d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.yetus.audience.InterfaceAudience; @@ -183,6 +184,13 @@ public static List toStoreFileInfo(Collection storefi return storefiles.stream().map(HStoreFile::getFileInfo).collect(Collectors.toList()); } + public static List toHStoreFile(List storeFileInfoList, + BloomType bloomType, CacheConfig cacheConf) { + return storeFileInfoList.stream() + .map(storeFileInfo -> new HStoreFile(storeFileInfo, bloomType, cacheConf)) + .collect(Collectors.toList()); + } + public static long getTotalUncompressedBytes(List files) { return files.stream() .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index b000d837d59b..047d4923ed74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -29,13 +30,20 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference.Range; import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry; import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; @@ -107,8 +115,18 @@ public boolean requireWritingToTmpDirFirst() { } private StoreFileEntry toStoreFileEntry(StoreFileInfo info) { - return StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize()) - .build(); + org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry.Builder entryBuilder = + StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize()); + if (info.isReference()) { + // TODO: Need a better way to fix the Protobuf generate enum Range to Reference.Range, + // otherwise it would result in DATA LOSS + org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference reference = + org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference.newBuilder() + .setSplitkey(ByteString.copyFrom(info.getReference().getSplitKey())) + .setRange(Range.forNumber(info.getReference().getFileRegion().ordinal())).build(); + entryBuilder.setReference(reference); + } + return entryBuilder.build(); } @Override @@ -119,7 +137,8 @@ protected void doAddNewStoreFiles(Collection newFiles) throws IOE builder.addStoreFile(toStoreFileEntry(info)); } for (StoreFileInfo info : newFiles) { - builder.addStoreFile(toStoreFileEntry(info)); + if (!storefiles.containsKey(info.getPath().getName())) + builder.addStoreFile(toStoreFileEntry(info)); } backedFile.update(builder); if (LOG.isTraceEnabled()) { @@ -176,4 +195,80 @@ protected void doSetStoreFiles(Collection files) throws IOExcepti } } } + + @Override + public Reference readReference(Path p) throws IOException { + String fileName = p.getName(); + StoreFileList list = backedFile.load(true); + for (StoreFileEntry entry : list.getStoreFileList()) { + if (entry.getName().equals(fileName)) { + return Reference.convert(entry.getReference()); + } + } + throw new FileNotFoundException("Reference does not exist for path : " + p); + } + + @Override + public boolean hasReferences() throws IOException { + StoreFileList list = backedFile.load(true); + for (StoreFileEntry entry : list.getStoreFileList()) { + if (entry.hasReference() || HFileLink.isHFileLink(entry.getName())) { + return true; + } + } + return false; + } + + @Override + public HFileLink createHFileLink(TableName linkedTable, String linkedRegion, String hfileName, + boolean createBackRef) throws IOException { + // String name = HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName); + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + HFileLink hfileLink = HFileLink.build(conf, linkedTable, linkedRegion, + ctx.getFamily().getNameAsString(), hfileName); + StoreFileInfo storeFileInfo = + new StoreFileInfo(conf, fs, new Path(ctx.getFamilyStoreDirectoryPath(), + HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName)), hfileLink); + // Path backRefPath = null; + if (createBackRef) { + // TODO: this should be done as part of commit + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, + ctx.getFamily().getNameAsString()); + Path backRefssDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); + fs.mkdirs(backRefssDir); + + // Create the reference for the link + String refName = HFileLink.createBackReferenceName(ctx.getTableName().toString(), + ctx.getRegionInfo().getEncodedName()); + Path backRefPath = new Path(backRefssDir, refName); + fs.createNewFile(backRefPath); + } + try { + // TODO do not add to SFT as of now + add(Collections.singletonList(storeFileInfo)); + } catch (Exception e) { + // LOG.error("couldn't create the link=" + name + " for " + ctx.getFamilyStoreDirectoryPath(), + // e); + // // Revert the reference if the link creation failed + // if (createBackRef) { + // fs.delete(backRefPath, false); + // } + } + return hfileLink; + } + + @Override + public Reference createReference(Reference reference, Path path) throws IOException { + // NOOP + return reference; + } + + @Override + public Reference createAndCommitReference(Reference reference, Path path) throws IOException { + StoreFileInfo storeFileInfo = + new StoreFileInfo(ctx.getRegionFileSystem().getFileSystem().getConf(), + ctx.getRegionFileSystem().getFileSystem(), path, reference); + add(Collections.singleton(storeFileInfo)); + return reference; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 595d5f4d1fc0..6a1816d0900f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStoreFile; @@ -102,6 +103,8 @@ void replace(Collection compactedFiles, Collection Reference createReference(Reference reference, Path path) throws IOException; + Reference createAndCommitReference(Reference reference, Path path) throws IOException; + /** * Reads the reference file from the given path. * @param path the {@link Path} to the reference file in the file system. @@ -131,7 +134,20 @@ StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryRepl * @return the file link name. * @throws IOException on file or parent directory creation failure. */ - String createHFileLink(final TableName linkedTable, final String linkedRegion, + HFileLink createHFileLink(final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException; + + /** + * Create a new HFileLink and add to SFT + *

+ * It also adds a back-reference to the hfile back-reference directory to simplify the + * reference-count and the cleaning process. + * @param hfileLinkName - HFileLink name (it contains hfile-region-table) + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return the file link name. + * @throws IOException on file or parent directory creation failure. + */ + HFileLink createAndCommitHFileLink(final TableName linkedTable, final String linkedRegion, final String hfileName, final boolean createBackRef) throws IOException; /** @@ -144,7 +160,7 @@ String createHFileLink(final TableName linkedTable, final String linkedRegion, * @return the file link name. * @throws IOException on file or parent directory creation failure. */ - String createFromHFileLink(final String hfileName, final boolean createBackRef) + HFileLink createFromHFileLink(final String hfileName, final boolean createBackRef) throws IOException; /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 779a114af594..a9f243033040 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -23,6 +23,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.regex.Matcher; @@ -219,6 +220,11 @@ public Reference createReference(Reference reference, Path path) throws IOExcept return reference; } + @Override + public Reference createAndCommitReference(Reference reference, Path path) throws IOException { + return createReference(reference, path); + } + /** * Returns true if the specified family has reference files * @param familyName Column Family Name @@ -324,7 +330,19 @@ public StoreFileInfo getStoreFileInfo(FileStatus fileStatus, Path initialPath, isPrimaryReplica); } - public String createHFileLink(final TableName linkedTable, final String linkedRegion, + public HFileLink createAndCommitHFileLink(final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException { + HFileLink hFileLink = createHFileLink(linkedTable, linkedRegion, hfileName, createBackRef); + Path path = new Path(ctx.getFamilyStoreDirectoryPath(), + HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName)); + StoreFileInfo storeFileInfo = + new StoreFileInfo(conf, this.ctx.getRegionFileSystem().getFileSystem(), path, hFileLink); + add(Arrays.asList(storeFileInfo)); + System.out.println("Created hfilelink " + path.toString()); + return hFileLink; + } + + public HFileLink createHFileLink(final TableName linkedTable, final String linkedRegion, final String hfileName, final boolean createBackRef) throws IOException { String name = HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName); String refName = HFileLink.createBackReferenceName(ctx.getTableName().toString(), @@ -349,7 +367,8 @@ public String createHFileLink(final TableName linkedTable, final String linkedRe try { // Create the link if (fs.createNewFile(new Path(ctx.getFamilyStoreDirectoryPath(), name))) { - return name; + return new HFileLink(new Path(ctx.getFamilyStoreDirectoryPath(), name), backRefPath, null, + archiveStoreDir); } } catch (IOException e) { LOG.error("couldn't create the link=" + name + " for " + ctx.getFamilyStoreDirectoryPath(), @@ -365,7 +384,7 @@ public String createHFileLink(final TableName linkedTable, final String linkedRe } - public String createFromHFileLink(final String hfileLinkName, final boolean createBackRef) + public HFileLink createFromHFileLink(final String hfileLinkName, final boolean createBackRef) throws IOException { Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(hfileLinkName); if (!m.matches()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 828f1974fca7..a2410238bb3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; @@ -118,7 +119,15 @@ public static Class getTrackerClass(String trackerNa public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = getTrackerClass(conf); + Class tracker; + if ( + ctx != null && ctx.getRegionInfo().getEncodedName() + .equals(MobUtils.getMobRegionInfo(ctx.getTableName()).getEncodedName()) + ) { + tracker = Trackers.DEFAULT.clazz; + } else { + tracker = getTrackerClass(conf); + } LOG.debug("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 3f01432472de..4237e65f537d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -465,7 +467,8 @@ public void editRegion(final RegionInfo hri) throws IOException { */ private void restoreRegion(final RegionInfo regionInfo, final SnapshotRegionManifest regionManifest) throws IOException { - restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName())); + restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()), + tableDir); } /** @@ -478,7 +481,8 @@ private void restoreMobRegion(final RegionInfo regionInfo, return; } restoreRegion(regionInfo, regionManifest, - MobUtils.getMobRegionPath(conf, tableDesc.getTableName())); + MobUtils.getMobRegionPath(conf, tableDesc.getTableName()), + MobUtils.getMobTableDir(conf, tableDesc.getTableName())); } /** @@ -486,26 +490,28 @@ private void restoreMobRegion(final RegionInfo regionInfo, * snapshot. */ private void restoreRegion(final RegionInfo regionInfo, - final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException { + final SnapshotRegionManifest regionManifest, Path regionDir, Path tableDir) throws IOException { Map> snapshotFiles = getRegionHFileReferences(regionManifest); String tableName = tableDesc.getTableName().getNameAsString(); final String snapshotName = snapshotDesc.getName(); - Path regionPath = new Path(tableDir, regionInfo.getEncodedName()); - HRegionFileSystem regionFS = (fs.exists(regionPath)) + // HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir, regionInfo); + HRegionFileSystem regionFS = (fs.exists(regionDir)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo); // Restore families present in the table for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { byte[] family = Bytes.toBytes(familyDir.getName()); - + ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.of(family); StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, - StoreContext.getBuilder().withColumnFamilyDescriptor(tableDesc.getColumnFamily(family)) + StoreContext.getBuilder().withColumnFamilyDescriptor(familyDescriptor) .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); - Set familyFiles = getTableRegionFamilyFiles(familyDir); + List storeFileInfos = tracker.load(); + List familyFiles = storeFileInfos.stream() + .map(storeFileInfo -> storeFileInfo.getPath().getName()).collect(Collectors.toList()); List snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); List filesToTrack = new ArrayList<>(); @@ -526,11 +532,13 @@ private void restoreRegion(final RegionInfo regionInfo, // Remove hfiles not present in the snapshot for (String hfileName : familyFiles) { - Path hfile = new Path(familyDir, hfileName); - if (!fs.getFileStatus(hfile).isDirectory()) { - LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName - + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); - HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); + for (StoreFileInfo storeFileInfo : storeFileInfos) { + if (hfileName.equals(storeFileInfo.getPath().getName())) { + tracker.removeStoreFiles( + StoreUtils.toHStoreFile(Collections.singletonList(storeFileInfo), null, null)); + LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName + + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); + } } } @@ -538,15 +546,17 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) { LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); - String fileName = + StoreFileInfo storeFileInfo = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); // mark the reference file to be added to tracker - filesToTrack.add(tracker.getStoreFileInfo(new Path(familyDir, fileName), true)); + filesToTrack.add(storeFileInfo); } } else { // Family doesn't exists in the snapshot LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); + LOG.debug("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family); fs.delete(familyDir, true); } @@ -571,29 +581,14 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) { LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName); - String fileName = + StoreFileInfo storeFileInfo = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); - files.add(tracker.getStoreFileInfo(new Path(familyDir, fileName), true)); + files.add(storeFileInfo); } tracker.set(files); } } - private Set getTableRegionFamilyFiles(final Path familyDir) throws IOException { - FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir); - if (hfiles == null) { - return Collections.emptySet(); - } - - Set familyFiles = new HashSet<>(hfiles.length); - for (int i = 0; i < hfiles.length; ++i) { - String hfileName = hfiles[i].getPath().getName(); - familyFiles.add(hfileName); - } - - return familyFiles; - } - /** * Clone specified regions. For each region create a new region and create a HFileLink for each * hfile. @@ -624,6 +619,9 @@ private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec, snapshotRegions.put(clonedRegionName, snapshotRegionInfo); } + // LOG.info("Print FS state right before creating new regions: " + "here is the tableDir- " + + // tableDir); + // CommonFSUtils.logFileSystemState(fs, rootDir, LOG); // create the regions on disk ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() { @@ -634,6 +632,9 @@ public void fillRegion(final HRegion region) throws IOException { } }); + // LOG.info("Print FS state right after creating new regions: " + "here is the tableDir- " + + // tableDir); + // CommonFSUtils.logFileSystemState(fs, rootDir, LOG); return clonedRegionsInfo; } @@ -648,6 +649,55 @@ private void cloneHdfsMobRegion(final Map region regionManifests.get(region.getEncodedName())); } + // private void cloneMobRegion(RegionInfo newRegionInfo, Path regionDir, RegionInfo + // snapshotRegionInfo, + // SnapshotRegionManifest manifest) throws IOException { + // final String tableName = tableDesc.getTableName().getNameAsString(); + // final String snapshotName = snapshotDesc.getName(); + // for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { + // Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); + // List clonedFiles = new ArrayList<>(); + // HRegionFileSystem regionFS = (fs.exists(regionDir)) + // ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) + // : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); + // + // Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, + // tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); + // + // StoreFileTracker tracker = + // StoreFileTrackerFactory + // .createMobSFT(sftConf, true, + // StoreContext.getBuilder().withFamilyStoreDirectoryPath(new Path(regionDir, + // familyFiles.getFamilyName().toStringUtf8())) + // .withRegionFileSystem(regionFS) + // .withColumnFamilyDescriptor( + // ColumnFamilyDescriptorBuilder.of(familyFiles.getFamilyName().toByteArray())) + // .build()); + // tracker.load(); + // for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { + // LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + // + snapshotName + " to table=" + tableName); + // if (MobUtils.isMobRegionInfo(newRegionInfo)) { + // String mobFileName = + // HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName()); + // Path mobPath = new Path(familyDir, mobFileName); + // if (fs.exists(mobPath)) { + // fs.delete(mobPath, true); + // } + // StoreFileInfo storeFileInfo = + // restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + // clonedFiles.add(storeFileInfo); + // } else { + // StoreFileInfo storeFileInfo = + // restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + // clonedFiles.add(storeFileInfo); + // } + // } + // tracker.add(clonedFiles); + // } + // + // } + /** * Clone region directory content from the snapshot info. Each region is encoded with the table * name, so the cloned region will have a different region name. Instead of copying the hfiles a @@ -661,8 +711,7 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); List clonedFiles = new ArrayList<>(); - Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); - HRegionFileSystem regionFS = (fs.exists(regionPath)) + HRegionFileSystem regionFS = (fs.exists(regionDir)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); @@ -671,11 +720,14 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, StoreFileTracker tracker = StoreFileTrackerFactory .create(sftConf, true, - StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir) + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath( + new Path(regionDir, familyFiles.getFamilyName().toStringUtf8())) .withRegionFileSystem(regionFS) .withColumnFamilyDescriptor( ColumnFamilyDescriptorBuilder.of(familyFiles.getFamilyName().toByteArray())) .build()); + tracker.load(); for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName); @@ -686,17 +738,16 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, if (fs.exists(mobPath)) { fs.delete(mobPath, true); } - restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + StoreFileInfo storeFileInfo = + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + clonedFiles.add(storeFileInfo); } else { - String file = + StoreFileInfo storeFileInfo = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); - clonedFiles.add(tracker.getStoreFileInfo(new Path(familyDir, file), true)); + clonedFiles.add(storeFileInfo); } } - // we don't need to track files under mobdir - if (!MobUtils.isMobRegionInfo(newRegionInfo)) { - tracker.set(clonedFiles); - } + tracker.add(clonedFiles); } } @@ -727,17 +778,23 @@ private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionIn * @param createBackRef - Whether back reference should be created. Defaults to true. * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ - private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, + private StoreFileInfo restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef, final StoreFileTracker tracker) throws IOException { String hfileName = storeFile.getName(); + StoreFileInfo info = null; if (HFileLink.isHFileLink(hfileName)) { - return tracker.createFromHFileLink(hfileName, createBackRef); + HFileLink hfileLink = tracker.createFromHFileLink(hfileName, createBackRef); + info = new StoreFileInfo(conf, fs, new Path(familyDir, hfileName), hfileLink); + return info; } else if (StoreFileInfo.isReference(hfileName)) { return restoreReferenceFile(familyDir, regionInfo, storeFile, tracker); } else { - return tracker.createHFileLink(regionInfo.getTable(), regionInfo.getEncodedName(), hfileName, - createBackRef); + HFileLink hfileLink = tracker.createHFileLink(regionInfo.getTable(), + regionInfo.getEncodedName(), hfileName, createBackRef); + return new StoreFileInfo(conf, fs, new Path(familyDir, HFileLink + .createHFileLinkName(regionInfo.getTable(), regionInfo.getEncodedName(), hfileName)), + hfileLink); } } @@ -764,10 +821,11 @@ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInf * @param regionInfo destination region info for the table * @param storeFile reference file name */ - private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, + private StoreFileInfo restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final StoreFileTracker tracker) throws IOException { String hfileName = storeFile.getName(); + StoreFileInfo storeFileInfo = null; // Extract the referred information (hfile name and parent region) Path refPath = @@ -800,11 +858,15 @@ private String restoreReferenceFile(final Path familyDir, final RegionInfo regio // Create the new reference if (storeFile.hasReference()) { Reference reference = Reference.convert(storeFile.getReference()); - tracker.createReference(reference, outPath); + tracker.createAndCommitReference(reference, outPath); + storeFileInfo = new StoreFileInfo(conf, fs, outPath, reference); } else { InputStream in; if (linkPath != null) { - in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs); + HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, linkPath); + storeFileInfo = new StoreFileInfo(conf, fs, outPath, hfileLink); + tracker.add(Collections.singletonList(storeFileInfo)); + in = hfileLink.open(fs); } else { linkPath = new Path(new Path( HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()), @@ -832,7 +894,7 @@ private String restoreReferenceFile(final Path familyDir, final RegionInfo regio daughters.setSecond(regionName); } } - return outPath.getName(); + return storeFileInfo; } /** @@ -861,6 +923,7 @@ private List getTableRegions() throws IOException { return null; } + System.out.println("No of regionDirs of this table " + tableDir + " is " + regionDirs.length); List regions = new ArrayList<>(regionDirs.length); for (int i = 0; i < regionDirs.length; ++i) { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); @@ -907,6 +970,8 @@ public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, File RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize. if (LOG.isDebugEnabled()) { + // LOG.debug("Root table dir:" + rootDir); + // CommonFSUtils.logFileSystemState(fs, rootDir, LOG); LOG.debug("Restored table dir:" + restoreDir); CommonFSUtils.logFileSystemState(fs, restoreDir, LOG); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java index 06dc4ec058b5..ecec70ebd66f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java @@ -129,6 +129,11 @@ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, } // else create a store file link. The link file does not exists on filesystem though. + // Path tableDir = CommonFSUtils.getTableDir(new Path("./"), regionInfoForFs.getTable()); + // Path initialPath = new Path(tableDir, + // new Path(regionInfoForFs.getEncodedName(), + // new Path(familyName, HFileLink.createHFileLinkName(regionInfoForFs.getTable(), + // regionInfoForFs.getEncodedName(), path.getName())))); if (HFileLink.isHFileLink(path) || StoreFileInfo.isHFile(path)) { HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); @@ -139,6 +144,10 @@ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, if (HFileLink.isHFileLink(referencePath)) { // HFileLink Reference HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, referencePath); + // initialPath = new Path(tableDir, + // new Path(regionInfoForFs.getEncodedName(), + // new Path(familyName, HFileLink.createHFileLinkName(regionInfoForFs.getTable(), + // regionInfoForFs.getEncodedName(), referencePath.getName())))); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference, link); } else { // Reference diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java index 7f1b7bd445c3..bfcaa00b8ffb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/CloneSnapshotFromClientAfterSplittingRegionTestBase.java @@ -27,12 +27,17 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.junit.Test; +import org.slf4j.LoggerFactory; public class CloneSnapshotFromClientAfterSplittingRegionTestBase extends CloneSnapshotFromClientTestBase { + private static final org.slf4j.Logger LOG = + LoggerFactory.getLogger(CloneSnapshotFromClientTestBase.class); + private void splitRegion() throws IOException { try (Table k = TEST_UTIL.getConnection().getTable(tableName); ResultScanner scanner = k.getScanner(new Scan())) { @@ -45,6 +50,7 @@ private void splitRegion() throws IOException { @Test public void testCloneSnapshotAfterSplittingRegion() throws IOException, InterruptedException { + System.out.println("Start - testCloneSnapshotAfterSplittingRegion"); // Turn off the CatalogJanitor admin.catalogJanitorSwitch(false); @@ -52,16 +58,31 @@ public void testCloneSnapshotAfterSplittingRegion() throws IOException, Interrup List regionInfos = admin.getRegions(tableName); RegionReplicaUtil.removeNonDefaultRegions(regionInfos); + System.out.println("Print FS state before split of the table: "); + CommonFSUtils.logFileSystemState(TEST_UTIL.getHBaseCluster().getMaster().getFileSystem(), + TEST_UTIL.getHBaseCluster().getMaster().getDataRootDir(), LOG); + // Split a region splitRegion(); + System.out.println("Print FS state before taking snapshot of the table: "); + CommonFSUtils.logFileSystemState(TEST_UTIL.getHBaseCluster().getMaster().getFileSystem(), + TEST_UTIL.getHBaseCluster().getMaster().getDataRootDir(), LOG); // Take a snapshot admin.snapshot(snapshotName2, tableName); - + Thread.currentThread().sleep(1000); + System.out.println("Print FS state after taking snapshot of the table: "); + CommonFSUtils.logFileSystemState(TEST_UTIL.getHBaseCluster().getMaster().getFileSystem(), + TEST_UTIL.getHBaseCluster().getMaster().getDataRootDir(), LOG); // Clone the snapshot to another table TableName clonedTableName = TableName.valueOf(getValidMethodName() + "-" + EnvironmentEdgeManager.currentTime()); admin.cloneSnapshot(snapshotName2, clonedTableName); + + System.out.println("Print FS state after cloning snapshot of the table: "); + CommonFSUtils.logFileSystemState(TEST_UTIL.getHBaseCluster().getMaster().getFileSystem(), + TEST_UTIL.getHBaseCluster().getMaster().getDataRootDir(), LOG); + SnapshotTestingUtils.waitForTableToBeOnline(TEST_UTIL, clonedTableName); verifyRowCount(TEST_UTIL, clonedTableName, snapshot1Rows); @@ -91,15 +112,21 @@ public void testCloneSnapshotAfterSplittingRegion() throws IOException, Interrup } } + System.out.println("Calling delete table"); + System.out.println("this testCloneSnapshotAfterSplittingRegion is successful!!"); + System.out + .println("End - testCloneSnapshotAfterSplittingRegion before finall/del and is successful"); TEST_UTIL.deleteTable(clonedTableName); } finally { admin.catalogJanitorSwitch(true); + System.out.println("End - testCloneSnapshotAfterSplittingRegion "); } } @Test public void testCloneSnapshotBeforeSplittingRegionAndDroppingTable() throws IOException, InterruptedException { + System.out.println("Start - testCloneSnapshotBeforeSplittingRegionAndDroppingTable"); // Turn off the CatalogJanitor admin.catalogJanitorSwitch(false); @@ -128,8 +155,13 @@ public void testCloneSnapshotBeforeSplittingRegionAndDroppingTable() SnapshotTestingUtils.waitForTableToBeOnline(TEST_UTIL, clonedTableName); verifyRowCount(TEST_UTIL, clonedTableName, snapshot1Rows); + System.out + .println("this testCloneSnapshotBeforeSplittingRegionAndDroppingTable is successful!!"); + System.out + .println("End before finally - testCloneSnapshotBeforeSplittingRegionAndDroppingTable"); } finally { admin.catalogJanitorSwitch(true); + System.out.println("End - testCloneSnapshotBeforeSplittingRegionAndDroppingTable"); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/RestoreSnapshotFromClientSchemaChangeTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/RestoreSnapshotFromClientSchemaChangeTestBase.java index c05818397c29..2315394b2206 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/RestoreSnapshotFromClientSchemaChangeTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/RestoreSnapshotFromClientSchemaChangeTestBase.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.MasterFileSystem; @@ -32,10 +33,15 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RestoreSnapshotFromClientSchemaChangeTestBase extends RestoreSnapshotFromClientTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(RestoreSnapshotFromClientSchemaChangeTestBase.class); + private Set getFamiliesFromFS(final TableName tableName) throws IOException { MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Set families = new HashSet<>(); @@ -74,8 +80,14 @@ public void testRestoreSchemaChange() throws Exception { admin.disableTable(tableName); admin.snapshot(snapshotName2, tableName); + System.out.println("check data immediately before restore : + with sft "); + CommonFSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()), + TEST_UTIL.getDefaultRootDirPath(), LOG); // Restore the snapshot (without the cf) admin.restoreSnapshot(snapshotName0); + System.out.println("check data immediately after restore : + with sft "); + CommonFSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()), + TEST_UTIL.getDefaultRootDirPath(), LOG); admin.enableTable(tableName); assertEquals(1, table.getDescriptor().getColumnFamilyCount()); try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index 621ecc087785..6c4d2d87ebdc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -213,6 +213,9 @@ public void testScanLimit() throws Exception { TableSnapshotScanner scanner = null; try { createTableAndSnapshot(UTIL, tableName, snapshotName, 50); + // LOG.info("Print FS state after taking snapshot of the table: "); + // CommonFSUtils.logFileSystemState(rootDir.getFileSystem(UTIL.getConfiguration()), rootDir, + // LOG); Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName); Scan scan = new Scan().withStartRow(bbb).setLimit(100); // limit the scan diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index 73b0f527981a..37b3de11bd4c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -429,7 +429,7 @@ private void testPrefetchWhenRefs(boolean compactionEnabled, Consumer .withRegionFileSystem(regionFS).build()); HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true, sft); Path ref = regionFS.splitStoreFile(region, "cf", file, fileWithSplitPoint.getSecond(), false, - new ConstantSizeRegionSplitPolicy(), sft); + new ConstantSizeRegionSplitPolicy(), sft).getPath(); conf.setBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, compactionEnabled); HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true, sft); refHsf.initReader(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java index 34341ce983d4..bc3b0d6a32fa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestPrefetchWithBucketCache.java @@ -197,7 +197,7 @@ public void testPrefetchRefsAfterSplit() throws Exception { byte[] splitPoint = RandomKeyValueUtil.randomOrderedKey(rand, 50); HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true, sft); Path ref = regionFS.splitStoreFile(region, "cf", file, splitPoint, false, - new ConstantSizeRegionSplitPolicy(), sft); + new ConstantSizeRegionSplitPolicy(), sft).getPath(); HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true, sft); // starts reader for the ref. The ref should resolve to the original file blocks // and not duplicate blocks in the cache. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java index 62dc4a3e188b..b6d9882b87b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Collections; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -34,8 +35,11 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -83,6 +87,8 @@ public class TestHFileLinkCleaner { private Path linkBackRef; private FileStatus[] backRefs; private HFileCleaner cleaner; + private StoreFileTracker sft; + private HFileLink hfileLink; private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); private static DirScanPool POOL; private static final long TTL = 1000; @@ -128,7 +134,7 @@ public void configureDirectoriesAndLinks() throws IOException { HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, CommonFSUtils.getTableDir(rootDir, tableLinkName), hriLink); - StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, + sft = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder() .withFamilyStoreDirectoryPath(new Path(regionFS.getRegionDir(), familyName)) .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(familyName)) @@ -145,8 +151,10 @@ private void createLink(StoreFileTracker sft, boolean createBackReference) throw // Create link to hfile familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, hriLink.getEncodedName(), familyName); fs.mkdirs(familyLinkPath); - hfileLinkName = + hfileLink = sft.createHFileLink(hri.getTable(), hri.getEncodedName(), hfileName, createBackReference); + hfileLinkName = hfileName; + linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); assertTrue(fs.exists(linkBackRefDir)); backRefs = fs.listStatus(linkBackRefDir); @@ -194,7 +202,19 @@ public void testHFileLinkByRemovingReference() throws Exception { assertTrue(fs.exists(hfilePath)); // simulate after removing the reference in data directory, the Link backref can be removed - fs.delete(new Path(familyLinkPath, hfileLinkName), false); + Path linkPath = new Path(familyLinkPath, + HFileLink.createHFileLinkName(hri.getTable(), hri.getEncodedName(), hfileName)); + HStoreFile storeFile = + new HStoreFile(new StoreFileInfo(conf, fs, linkPath, hfileLink), BloomType.NONE, null); + sft.removeStoreFiles(Collections.singletonList(storeFile)); + + // Delete attempts + // Path archiveLinkDir = HFileArchiveUtil.getStoreArchivePath(conf, tableLinkName, + // hriLink.getEncodedName(), familyName); + // Path archivedHFileLink = new Path(archiveLinkDir, hfileLinkName); + // fs.delete(archivedHFileLink, false); + // fs.delete(new Path(familyLinkPath, hfileLinkName), false); + cleaner.chore(); assertFalse("Link should be deleted", fs.exists(linkBackRef)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java index 1c586bbd10c7..89e29c0894d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java @@ -45,7 +45,7 @@ public TestMobCompactionOptMode(Boolean useFileBasedSFT) { protected void additonalConfigSetup() { conf.set(MobConstants.MOB_COMPACTION_TYPE_KEY, MobConstants.OPTIMIZED_MOB_COMPACTION_TYPE); - conf.setLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, 1000000); + conf.setLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, 10000000); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java index 3ad6585c4620..91f5a0ff1ea3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java @@ -43,9 +43,13 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.RegionSplitter; import org.junit.After; import org.junit.Before; @@ -148,6 +152,7 @@ public void setUp() throws Exception { private void loadData(TableName tableName, int num) { LOG.info("Started loading {} rows into {}", num, tableName); + System.out.println("Load started for " + tableName); try (final Table table = HTU.getConnection().getTable(tableName)) { for (int i = 0; i < num; i++) { byte[] key = new byte[32]; @@ -174,10 +179,18 @@ public void tearDown() throws Exception { @Test public void baseTestMobFileCompaction() throws InterruptedException, IOException { LOG.info("MOB compaction " + description() + " started"); + System.out.println("Calling loadAndFlush from base "); loadAndFlushThreeTimes(rows, table, famStr); + long filesAfterFlush = getNumberOfActualMobFiles(tableDescriptor, familyDescriptor, famStr); + LOG.info("Files after flush: {}", filesAfterFlush); mobCompact(tableDescriptor, familyDescriptor); - assertEquals("Should have 4 MOB files per region due to 3xflush + compaction.", numRegions * 4, - getNumberOfMobFiles(table, famStr)); + long filesAfterCompaction = + getNumberOfActualMobFiles(tableDescriptor, familyDescriptor, famStr); + LOG.info("Files after compaction: {}", filesAfterCompaction); + + assertEquals("Should have 1 MOB files per region due to 3xflush + compaction.", numRegions * 4, + getNumberOfActualMobFiles(tableDescriptor, tableDescriptor.getColumnFamily(famStr.getBytes()), + famStr)); cleanupAndVerifyCounts(table, famStr, 3 * rows); LOG.info("MOB compaction " + description() + " finished OK"); } @@ -191,14 +204,51 @@ public void testMobFileCompactionAfterSnapshotClone() throws InterruptedExceptio admin.snapshot(TestMobUtils.getTableName(test), table); admin.cloneSnapshot(TestMobUtils.getTableName(test), clone); assertEquals("Should have 3 hlinks per region in MOB area from snapshot clone", 3 * numRegions, - getNumberOfMobFiles(clone, famStr)); - mobCompact(admin.getDescriptor(clone), familyDescriptor); + getNumberOfStoreFiles(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr)); + mobCompact(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes())); assertEquals("Should have 3 hlinks + 1 MOB file per region due to clone + compact", - 4 * numRegions, getNumberOfMobFiles(clone, famStr)); + 4 * numRegions, getNumberOfActualMobFiles(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr)); cleanupAndVerifyCounts(clone, famStr, 3 * rows); LOG.info("MOB compaction of cloned snapshot, " + description() + " finished OK"); } + // protected long getNumberOfActualMobFiles(TableDescriptor descriptor, ColumnFamilyDescriptor + // familyDesc, String family) throws IOException { + // FileSystem fs = FileSystem.get(conf); + // long totalFiles = 0; + // HRegionFileSystem regionFS = (fs.exists(MobUtils.getMobRegionPath(conf, + // descriptor.getTableName())) + // ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, MobUtils.getMobTableDir(conf, + // descriptor.getTableName()), MobUtils.getMobRegionInfo(descriptor.getTableName()), + // false) + // : HRegionFileSystem.createRegionOnFileSystem(conf, fs, MobUtils.getMobTableDir(conf, + // descriptor.getTableName()), MobUtils.getMobRegionInfo(descriptor.getTableName()))); + // StoreFileTracker sft = StoreFileTrackerFactory.createMobSFT(conf, + // false,StoreContext.getBuilder().withColumnFamilyDescriptor(familyDesc) + // .withRegionFileSystem(regionFS) + // .withFamilyStoreDirectoryPath(MobUtils.getMobFamilyPath(conf, descriptor.getTableName(), + // family)) + // .build()); + // totalFiles += sft.load().size(); + // return totalFiles; + // } + + protected long getNumberOfActualMobFiles(TableDescriptor descriptor, + ColumnFamilyDescriptor familyDesc, String family) throws IOException { + FileSystem fs = FileSystem.get(conf); + Path dir = MobUtils.getMobFamilyPath(conf, descriptor.getTableName(), family); + FileStatus[] stat = fs.listStatus(dir); + for (FileStatus st : stat) { + LOG.debug("MOB Directory content: {}", st.getPath()); + } + LOG.debug("MOB Directory content total files: {}", stat.length); + + return stat.length; + } + @Test public void testMobFileCompactionAfterSnapshotCloneAndFlush() throws InterruptedException, IOException { @@ -208,25 +258,57 @@ public void testMobFileCompactionAfterSnapshotCloneAndFlush() LOG.debug("Taking snapshot and cloning table {}", table); admin.snapshot(TestMobUtils.getTableName(test), table); admin.cloneSnapshot(TestMobUtils.getTableName(test), clone); + System.out.println("SFT config - " + useFileBasedSFT); + long num = getNumberOfStoreFiles(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr); + System.out.println("Should have 3 hlinks per region in MOB area from snapshot clone " + + 3 * numRegions + " " + num); + System.out.println("check mob data immediately after cloning : + with sft " + useFileBasedSFT); + CommonFSUtils.logFileSystemState(FileSystem.get(conf), HTU.getDefaultRootDirPath(), LOG); assertEquals("Should have 3 hlinks per region in MOB area from snapshot clone", 3 * numRegions, - getNumberOfMobFiles(clone, famStr)); + getNumberOfStoreFiles(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr)); loadAndFlushThreeTimes(rows, clone, famStr); - mobCompact(admin.getDescriptor(clone), familyDescriptor); + mobCompact(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes())); + CommonFSUtils.logFileSystemState(FileSystem.get(conf), HTU.getDefaultRootDirPath(), LOG); assertEquals("Should have 7 MOB file per region due to clone + 3xflush + compact", - 7 * numRegions, getNumberOfMobFiles(clone, famStr)); + 7 * numRegions, getNumberOfActualMobFiles(admin.getDescriptor(clone), + admin.getDescriptor(clone).getColumnFamily(famStr.getBytes()), famStr)); cleanupAndVerifyCounts(clone, famStr, 6 * rows); LOG.info("MOB compaction of cloned snapshot w flush, " + description() + " finished OK"); } + // private void getNumberOfStoreFiles(TableDescriptor descriptor, ColumnFamilyDescriptor + // columnFamily, String famstr2, + // boolean b) throws IOException { + // List regions = HTU.getHBaseCluster().getRegions(descriptor.getTableName()); + // System.out.println("Number of regions here - " + regions.size()); + // for (HRegion region : regions) { + // HStore store = region.getStore(columnFamily.getName()); + // // This counts regular region files (with MOB references) via SFT + // StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, store.getStoreContext()); + // if(sft.load().size() > 1) { + // System.out.println( " loading files of region name - + " + + // region.getRegionInfo().getEncodedName() + " " + sft.load()); + // break; + // } + // } + // } + protected void loadAndFlushThreeTimes(int rows, TableName table, String family) throws IOException { - final long start = getNumberOfMobFiles(table, family); + final long start = getNumberOfActualMobFiles(admin.getDescriptor(table), + admin.getDescriptor(table).getColumnFamily(family.getBytes()), family); // Load and flush data 3 times loadData(table, rows); loadData(table, rows); loadData(table, rows); + System.out.println("FS state right before assert of flushes"); + CommonFSUtils.logFileSystemState(FileSystem.get(conf), HTU.getDefaultRootDirPath(), LOG); assertEquals("Should have 3 more mob files per region from flushing.", start + numRegions * 3, - getNumberOfMobFiles(table, family)); + getNumberOfActualMobFiles(admin.getDescriptor(table), + admin.getDescriptor(table).getColumnFamily(family.getBytes()), family)); } protected String description() { @@ -293,7 +375,8 @@ protected void cleanupAndVerifyCounts(TableName table, String family, int rows) } assertEquals("After cleaning, we should have 1 MOB file per region based on size.", numRegions, - getNumberOfMobFiles(table, family)); + getNumberOfActualMobFiles(admin.getDescriptor(table), + admin.getDescriptor(table).getColumnFamily(family.getBytes()), family)); LOG.debug("checking count of rows"); long scanned = scanTable(table); @@ -301,18 +384,6 @@ protected void cleanupAndVerifyCounts(TableName table, String family, int rows) } - protected long getNumberOfMobFiles(TableName tableName, String family) throws IOException { - FileSystem fs = FileSystem.get(conf); - Path dir = MobUtils.getMobFamilyPath(conf, tableName, family); - FileStatus[] stat = fs.listStatus(dir); - for (FileStatus st : stat) { - LOG.debug("MOB Directory content: {}", st.getPath()); - } - LOG.debug("MOB Directory content total files: {}", stat.length); - - return stat.length; - } - protected long scanTable(TableName tableName) { try (final Table table = HTU.getConnection().getTable(tableName); final ResultScanner scanner = table.getScanner(fam)) { @@ -333,4 +404,18 @@ protected long scanTable(TableName tableName) { } return 0; } + + protected long getNumberOfStoreFiles(TableDescriptor descriptor, + ColumnFamilyDescriptor familyDesc, String family) throws IOException { + List regions = HTU.getHBaseCluster().getRegions(descriptor.getTableName()); + long totalFiles = 0; + System.out.println("Number of regions here - " + regions.size()); + for (HRegion region : regions) { + HStore store = region.getStore(familyDesc.getName()); + // This counts regular region files (with MOB references) via SFT + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, store.getStoreContext()); + totalFiles += sft.load().size(); + } + return totalFiles; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java index cc9455641cbb..2d7d4b92a2e2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java @@ -91,7 +91,7 @@ public void testSplitStoreDir() throws Exception { StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), true, region.getStores().get(0).getStoreContext()); Path result = regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy(), sft); + Bytes.toBytes("002"), false, region.getSplitPolicy(), sft).getPath(); // asserts the reference file naming is correct validateResultingFile(region.getRegionInfo().getEncodedName(), result); // Additionally check if split region dir was created directly under table dir, not on .tmp @@ -170,13 +170,13 @@ public void testCommitDaughterRegionWithFiles() throws Exception { Path splitDirA = regionFS.getSplitsDir(daughterA); Path splitDirB = regionFS.getSplitsDir(daughterB); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - List filesA = new ArrayList<>(); + List filesA = new ArrayList<>(); StoreFileTracker sft = StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), true, region.getStores().get(0).getStoreContext()); filesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, Bytes.toBytes("002"), false, region.getSplitPolicy(), sft)); - List filesB = new ArrayList<>(); + List filesB = new ArrayList<>(); filesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, Bytes.toBytes("002"), true, region.getSplitPolicy(), sft)); MasterProcedureEnv env = @@ -217,7 +217,7 @@ public void testCommitMergedRegion() throws Exception { true, first.getStore(FAMILY_NAME).getStoreContext())); // merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - List mergedFiles = new ArrayList<>(); + List mergedFiles = new ArrayList<>(); mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file, StoreFileTrackerFactory .create(configuration, true, second.getStore(FAMILY_NAME).getStoreContext()))); MasterProcedureEnv env = @@ -241,11 +241,11 @@ private void waitForSplitProcComplete(int attempts, int waitTime) throws Excepti } } - private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, + private StoreFileInfo mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, HStoreFile file, StoreFileTracker sft) throws IOException { - Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), + StoreFileInfo mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file, sft); - validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile); + validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile.getPath()); return mergedFile; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index 3647a4e47ad6..ece237d0f033 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -66,6 +66,9 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl; import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequester; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -193,6 +196,10 @@ public void setUp() throws Exception { when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1)); when(rss.getConfiguration()).thenReturn(CONF); when(rss.getRegionServerAccounting()).thenReturn(new RegionServerAccounting(CONF)); + when(rss.getRegionServerSpaceQuotaManager()).thenReturn(null); // or mock it properly + when(rss.getFlushRequester()).thenReturn(mock(FlushRequester.class)); + when(rss.getCompactionRequestor()).thenReturn(mock(CompactionRequester.class)); + when(rss.getMetrics()).thenReturn(mock(MetricsRegionServer.class)); String string = org.apache.hadoop.hbase.executor.EventType.RS_COMPACTED_FILES_DISCHARGER.toString(); ExecutorService es = new ExecutorService(string); @@ -357,6 +364,7 @@ public void testReplayFlushesAndCompactions() throws IOException { // compaction from primary LOG.info("-- Compacting primary, only 1 store"); primaryRegion.compactStore(Bytes.toBytes("cf1"), NoLimitThroughputController.INSTANCE); + System.out.println("primary region name - " + primaryRegion.getRegionInfo().getEncodedName()); // now replay the edits and the flush marker reader = createWALReaderForPrimary(); @@ -417,14 +425,26 @@ public void testReplayFlushesAndCompactions() throws IOException { // after replay verify that everything is still visible verifyData(secondaryRegion, 0, lastReplayed + 1, cq, families); } else if (compactionDesc != null) { + System.out + .println("secondary region name - " + secondaryRegion.getRegionInfo().getEncodedName()); secondaryRegion.replayWALCompactionMarker(compactionDesc, true, false, Long.MAX_VALUE); // assert that the compaction is applied for (HStore store : secondaryRegion.getStores()) { + StoreFileTracker sft = + StoreFileTrackerFactory.create(CONF, false, store.getStoreContext()); + HStore primatryStore = primaryRegion.getStore("cf1".getBytes()); + StoreFileTracker primarysft = + StoreFileTrackerFactory.create(CONF, false, primatryStore.getStoreContext()); + System.out + .println("is it actually marked primary - " + primatryStore.isPrimaryReplicaStore()); + System.out + .println("files in primary sft load post compaction - " + primarysft.load().size()); if (store.getColumnFamilyName().equals("cf1")) { + System.out.println("files in secondary region - " + sft.load()); assertEquals(1, store.getStorefilesCount()); } else { - assertEquals(expectedStoreFileCount, store.getStorefilesCount()); + assertEquals(expectedStoreFileCount, sft.load().size()); } } } else { @@ -1538,17 +1558,17 @@ public void testReplayBulkLoadEvent() throws IOException { // replay the bulk load event secondaryRegion.replayWALBulkLoadEventMarker(bulkloadEvent); - List storeFileName = new ArrayList<>(); + List storeFileNames = new ArrayList<>(); for (StoreDescriptor storeDesc : bulkloadEvent.getStoresList()) { - storeFileName.addAll(storeDesc.getStoreFileList()); + storeFileNames.addAll(storeDesc.getStoreFileList()); } // assert that the bulk loaded files are picked for (HStore s : secondaryRegion.getStores()) { for (HStoreFile sf : s.getStorefiles()) { - storeFileName.remove(sf.getPath().getName()); + storeFileNames.remove(sf.getPath().getName()); } } - assertTrue("Found some store file isn't loaded:" + storeFileName, storeFileName.isEmpty()); + assertTrue("Found some store file isn't loaded:" + storeFileNames, storeFileNames.isEmpty()); LOG.info("-- Verifying edits from secondary"); for (byte[] family : families) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index 69f549dd6b92..1580d1e690e6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -1171,12 +1171,12 @@ public void testCacheOnWriteEvictOnClose() throws Exception { private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo hri, final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef, StoreFileTracker sft) throws IOException { - Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null, sft); - if (null == path) { + StoreFileInfo sfi = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null, sft); + if (null == sfi) { return null; } - List splitFiles = new ArrayList<>(); - splitFiles.add(path); + List splitFiles = new ArrayList<>(); + splitFiles.add(sfi); MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class); MasterServices mockServices = mock(MasterServices.class); when(mockEnv.getMasterServices()).thenReturn(mockServices); @@ -1187,7 +1187,7 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); when(mockTblDescs.get(any())).thenReturn(mockTblDesc); Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); - return new Path(new Path(regionDir, family), path.getName()); + return new Path(new Path(regionDir, family), sfi.getPath().getName()); } private StoreFileWriter writeStoreFile(Configuration conf, CacheConfig cacheConf, Path path, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index 47790dd20da4..59f16f56dca8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -122,7 +122,7 @@ public void testCommitDaughterRegion() throws Exception { .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false) .setRegionId(region.getRegionInfo().getRegionId()).build(); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - List splitFilesA = new ArrayList<>(); + List splitFilesA = new ArrayList<>(); HRegionFileSystem regionFs = region.getRegionFileSystem(); StoreFileTracker sft = StoreFileTrackerFactory.create(region.getBaseConf(), true, StoreContext.getBuilder() @@ -130,7 +130,7 @@ public void testCommitDaughterRegion() throws Exception { .withRegionFileSystem(regionFs).build()); splitFilesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, Bytes.toBytes("002"), false, region.getSplitPolicy(), sft)); - List splitFilesB = new ArrayList<>(); + List splitFilesB = new ArrayList<>(); splitFilesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, Bytes.toBytes("002"), true, region.getSplitPolicy(), sft)); MasterProcedureEnv env = @@ -164,7 +164,7 @@ public void testCommitMergedRegion() throws Exception { TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); - List mergedFiles = new ArrayList<>(); + List mergedFiles = new ArrayList<>(); // merge file from first region mergedFiles.add(mergeFileFromRegion(first, mergeFS)); // merge file from second region @@ -256,7 +256,7 @@ private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Excepti } } - private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) + private StoreFileInfo mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) throws IOException { HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; HRegionFileSystem regionFs = regionToMerge.getRegionFileSystem(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 77bd2a29f4ae..9b18aa69d14c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -957,11 +957,13 @@ public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck() HRegionFileSystem hfs = region.getRegionFileSystem(); StoreFileTracker sft = StoreFileTrackerFactory.create(TESTING_UTIL.getConfiguration(), true, store.getStoreContext()); - Path referencePath = hfs.splitStoreFile(region.getRegionInfo(), "f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft); + Path referencePath = + hfs.splitStoreFile(region.getRegionInfo(), "f", storefiles.iterator().next(), + Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft).getPath(); assertNull(referencePath); - referencePath = hfs.splitStoreFile(region.getRegionInfo(), "i_f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft); + referencePath = + hfs.splitStoreFile(region.getRegionInfo(), "i_f", storefiles.iterator().next(), + Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft).getPath(); assertNotNull(referencePath); } finally { TESTING_UTIL.deleteTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java index ffc4e17f6f8b..26ddffe5500a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java @@ -93,10 +93,10 @@ public void testEqualsWithLink() throws IOException { HFileLink link2 = new HFileLink(new Path(origin, "f1"), new Path(tmp, "f1"), new Path(mob, "f1"), new Path(archive, "f1")); - StoreFileInfo info1 = - new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link1); - StoreFileInfo info2 = - new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link2); + StoreFileInfo info1 = new StoreFileInfo(TEST_UTIL.getConfiguration(), + TEST_UTIL.getTestFileSystem(), new Path(archive, "f1"), link1); + StoreFileInfo info2 = new StoreFileInfo(TEST_UTIL.getConfiguration(), + TEST_UTIL.getTestFileSystem(), new Path(archive, "f1"), link2); assertEquals(info1, info2); assertEquals(info1.hashCode(), info2.hashCode());