From 9897f4fa7a2aa8db5422089d2841c2aa6690a048 Mon Sep 17 00:00:00 2001 From: William Lo Date: Fri, 18 Oct 2024 16:23:43 -0400 Subject: [PATCH 1/7] WIP --- .../copy/hive/HiveDatasetFinder.java | 13 ++++++- .../copy/hive/HiveDatasetFinderTest.java | 35 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index a73ae704865..e4358d9799a 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; +import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -80,6 +81,8 @@ public class HiveDatasetFinder implements IterableDatasetFinder { public static final String DEFAULT_TABLE_PATTERN = "*"; public static final String TABLE_FILTER = HIVE_DATASET_PREFIX + ".tableFilter"; + public static final String TABLE_FOLDER_FILTER = HIVE_DATASET_PREFIX + ".tableFolderFilter"; + /* * By setting the prefix, only config keys with this prefix will be used to build a HiveDataset. * By passing scoped configurations the same config keys can be used in different contexts. @@ -118,6 +121,8 @@ public class HiveDatasetFinder implements IterableDatasetFinder { protected final Function configStoreDatasetUriBuilder; protected final Optional> tableFilter; + protected final Optional tableFolderFilter; + protected final String datasetConfigPrefix; protected final ConfigClient configClient; private final Config jobConfig; @@ -194,6 +199,9 @@ protected HiveDatasetFinder(FileSystem fs, Properties properties, HiveMetastoreC } else { this.tableFilter = Optional.absent(); } + if (StringUtils.isNotEmpty(properties.getProperty(TABLE_FOLDER_FILTER))) { + this.tableFolderFilter = Optional.of(properties.getProperty(TABLE_FOLDER_FILTER)); + } } protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { @@ -262,7 +270,7 @@ protected HiveDataset computeNext() { try (AutoReturnableObject client = HiveDatasetFinder.this.clientPool.getClient()) { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); - if (tableFilter.isPresent() && !tableFilter.get().apply(table)) { + if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || tableFolderFilter.isPresent() && shouldFilterTableLocation(tableFolderFilter, table)) { continue; } @@ -294,6 +302,9 @@ protected HiveDataset computeNext() { }; } + protected static boolean shouldFilterTableLocation(Optional regex, Table table) { + return Pattern.compile(regex.get()).matcher(table.getSd().getLocation()).matches(); + } /** * @deprecated Use {@link #createHiveDataset(Table, Config)} instead diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java index 07945202551..3725213e486 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java @@ -215,6 +215,41 @@ public void testDatasetConfig() throws Exception { } + @Test + public void testDatasetConfig() throws Exception { + + List dbAndTables = Lists.newArrayList(); + dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1")); + HiveMetastoreClientPool pool = getTestPool(dbAndTables); + + Properties properties = new Properties(); + properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, ""); + + properties.put("hive.dataset.test.conf1", "conf1-val1"); + properties.put("hive.dataset.test.conf2", "conf2-val2"); + + HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); + List datasets = Lists.newArrayList(finder.getDatasetsIterator()); + + Assert.assertEquals(datasets.size(), 1); + HiveDataset hiveDataset = datasets.get(0); + + Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf1"), "conf1-val1"); + Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf2"), "conf2-val2"); + + // Test scoped configs with prefix + properties.put(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, "hive.dataset.test"); + + finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); + datasets = Lists.newArrayList(finder.getDatasetsIterator()); + + Assert.assertEquals(datasets.size(), 1); + hiveDataset = datasets.get(0); + Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf1"), "conf1-val1"); + Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf2"), "conf2-val2"); + + } + private HiveMetastoreClientPool getTestPool(List dbAndTables) throws Exception { SetMultimap entities = HashMultimap.create(); From f556f27dc4a1397416a97e8d6163e306fcf8c683 Mon Sep 17 00:00:00 2001 From: William Lo Date: Fri, 18 Oct 2024 23:33:09 -0400 Subject: [PATCH 2/7] Add regex filter for table based on location --- .../copy/hive/HiveDatasetFinder.java | 30 ++++++++++--------- .../copy/hive/HiveDatasetFinderTest.java | 23 +++++--------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index e4358d9799a..48a5977135a 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -17,7 +17,6 @@ package org.apache.gobblin.data.management.copy.hive; -import com.google.common.base.Throwables; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; @@ -27,12 +26,6 @@ import java.util.Properties; import java.util.regex.Pattern; -import javax.annotation.Nonnull; - -import lombok.Data; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.ConstructorUtils; import org.apache.hadoop.fs.FileSystem; @@ -44,12 +37,18 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import javax.annotation.Nonnull; +import lombok.Data; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.config.client.ConfigClient; import org.apache.gobblin.config.client.ConfigClientCache; import org.apache.gobblin.config.client.ConfigClientUtils; @@ -81,7 +80,8 @@ public class HiveDatasetFinder implements IterableDatasetFinder { public static final String DEFAULT_TABLE_PATTERN = "*"; public static final String TABLE_FILTER = HIVE_DATASET_PREFIX + ".tableFilter"; - public static final String TABLE_FOLDER_FILTER = HIVE_DATASET_PREFIX + ".tableFolderFilter"; + // Property used to filter tables only physically within a folder, represented by a regex + public static final String TABLE_FOLDER_ALLOWLIST_FILTER = HIVE_DATASET_PREFIX + ".tableFolderAllowlistFilter"; /* * By setting the prefix, only config keys with this prefix will be used to build a HiveDataset. @@ -121,7 +121,7 @@ public class HiveDatasetFinder implements IterableDatasetFinder { protected final Function configStoreDatasetUriBuilder; protected final Optional> tableFilter; - protected final Optional tableFolderFilter; + protected final Optional tableFolderAllowlistRegex; protected final String datasetConfigPrefix; protected final ConfigClient configClient; @@ -199,9 +199,8 @@ protected HiveDatasetFinder(FileSystem fs, Properties properties, HiveMetastoreC } else { this.tableFilter = Optional.absent(); } - if (StringUtils.isNotEmpty(properties.getProperty(TABLE_FOLDER_FILTER))) { - this.tableFolderFilter = Optional.of(properties.getProperty(TABLE_FOLDER_FILTER)); - } + this.tableFolderAllowlistRegex = properties.containsKey(TABLE_FOLDER_ALLOWLIST_FILTER) ? + Optional.of(properties.getProperty(TABLE_FOLDER_ALLOWLIST_FILTER)): Optional.absent(); } protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { @@ -270,7 +269,7 @@ protected HiveDataset computeNext() { try (AutoReturnableObject client = HiveDatasetFinder.this.clientPool.getClient()) { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); - if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || tableFolderFilter.isPresent() && shouldFilterTableLocation(tableFolderFilter, table)) { + if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { continue; } @@ -302,7 +301,10 @@ protected HiveDataset computeNext() { }; } - protected static boolean shouldFilterTableLocation(Optional regex, Table table) { + protected static boolean shouldAllowTableLocation(Optional regex, Table table) { + if (!regex.isPresent()) { + return true; + } return Pattern.compile(regex.get()).matcher(table.getSd().getLocation()).matches(); } diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java index 3725213e486..25b06201bec 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java @@ -216,38 +216,29 @@ public void testDatasetConfig() throws Exception { } @Test - public void testDatasetConfig() throws Exception { - + public void testHiveTableFolderFilter() throws Exception { List dbAndTables = Lists.newArrayList(); dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1")); HiveMetastoreClientPool pool = getTestPool(dbAndTables); Properties properties = new Properties(); properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, ""); - - properties.put("hive.dataset.test.conf1", "conf1-val1"); - properties.put("hive.dataset.test.conf2", "conf2-val2"); + // Try a regex with multiple groups + properties.put(HiveDatasetFinder.TABLE_FOLDER_ALLOWLIST_FILTER, "(/tmp/|a).*"); HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); List datasets = Lists.newArrayList(finder.getDatasetsIterator()); Assert.assertEquals(datasets.size(), 1); - HiveDataset hiveDataset = datasets.get(0); - - Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf1"), "conf1-val1"); - Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf2"), "conf2-val2"); - // Test scoped configs with prefix - properties.put(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, "hive.dataset.test"); + properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, ""); + // The table located at /tmp/test should be filtered + properties.put(HiveDatasetFinder.TABLE_FOLDER_ALLOWLIST_FILTER, "/a/b"); finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); datasets = Lists.newArrayList(finder.getDatasetsIterator()); - Assert.assertEquals(datasets.size(), 1); - hiveDataset = datasets.get(0); - Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf1"), "conf1-val1"); - Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf2"), "conf2-val2"); - + Assert.assertEquals(datasets.size(), 0); } private HiveMetastoreClientPool getTestPool(List dbAndTables) throws Exception { From e1ed1a6ee272e2f8142460cf21e3b114d06ffc0a Mon Sep 17 00:00:00 2001 From: William Lo Date: Fri, 18 Oct 2024 23:34:09 -0400 Subject: [PATCH 3/7] Cleanup --- .../gobblin/data/management/copy/hive/HiveDatasetFinder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index 48a5977135a..a51f951f279 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -269,7 +269,8 @@ protected HiveDataset computeNext() { try (AutoReturnableObject client = HiveDatasetFinder.this.clientPool.getClient()) { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); - if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { + if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) + || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { continue; } From d4e60bef2e0359c853fd00d3441818ad6d975542 Mon Sep 17 00:00:00 2001 From: William Lo Date: Sat, 19 Oct 2024 00:20:33 -0400 Subject: [PATCH 4/7] Add logs --- .../gobblin/data/management/copy/hive/HiveDatasetFinder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index a51f951f279..ad8153dd71c 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -271,6 +271,7 @@ protected HiveDataset computeNext() { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { + log.info("Ignoring table {} as its underlying location does not part of allowlist regex {}", dbAndTable, tableFolderAllowlistRegex); continue; } From e736bab9beb1e1ce61a7f9cb3964b76d71ea23b2 Mon Sep 17 00:00:00 2001 From: William Lo Date: Sun, 20 Oct 2024 21:47:34 -0400 Subject: [PATCH 5/7] add more details to log --- .../gobblin/data/management/copy/hive/HiveDatasetFinder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index ad8153dd71c..9c99ba2b768 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -271,7 +271,8 @@ protected HiveDataset computeNext() { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { - log.info("Ignoring table {} as its underlying location does not part of allowlist regex {}", dbAndTable, tableFolderAllowlistRegex); + log.info("Ignoring table {} as its underlying location {} does not part of allowlist regex {}", dbAndTable, + table.getSd().getLocation(), tableFolderAllowlistRegex); continue; } From da1902d94073155b93a00b317e00e3be63f8ccea Mon Sep 17 00:00:00 2001 From: William Lo Date: Mon, 21 Oct 2024 16:51:28 -0400 Subject: [PATCH 6/7] Fix typo --- .../gobblin/data/management/copy/hive/HiveDatasetFinder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index 9c99ba2b768..fa48f89e24c 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -271,7 +271,7 @@ protected HiveDataset computeNext() { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { - log.info("Ignoring table {} as its underlying location {} does not part of allowlist regex {}", dbAndTable, + log.info("Ignoring table {} as its underlying location {} does not pass allowlist regex {}", dbAndTable, table.getSd().getLocation(), tableFolderAllowlistRegex); continue; } From 19062867ffc93aeb0e0e7729a8858f8a517f2fae Mon Sep 17 00:00:00 2001 From: William Lo Date: Mon, 21 Oct 2024 23:02:46 -0400 Subject: [PATCH 7/7] Address reviews --- .../copy/hive/HiveDatasetFinder.java | 10 ++++----- .../copy/hive/HiveDatasetFinderTest.java | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java index fa48f89e24c..b3a3c846e92 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java @@ -121,7 +121,7 @@ public class HiveDatasetFinder implements IterableDatasetFinder { protected final Function configStoreDatasetUriBuilder; protected final Optional> tableFilter; - protected final Optional tableFolderAllowlistRegex; + protected final Optional tableFolderAllowlistRegex; protected final String datasetConfigPrefix; protected final ConfigClient configClient; @@ -200,7 +200,7 @@ protected HiveDatasetFinder(FileSystem fs, Properties properties, HiveMetastoreC this.tableFilter = Optional.absent(); } this.tableFolderAllowlistRegex = properties.containsKey(TABLE_FOLDER_ALLOWLIST_FILTER) ? - Optional.of(properties.getProperty(TABLE_FOLDER_ALLOWLIST_FILTER)): Optional.absent(); + Optional.of(Pattern.compile(properties.getProperty(TABLE_FOLDER_ALLOWLIST_FILTER))): Optional.absent(); } protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { @@ -272,7 +272,7 @@ protected HiveDataset computeNext() { if ((tableFilter.isPresent() && !tableFilter.get().apply(table)) || !shouldAllowTableLocation(tableFolderAllowlistRegex, table)) { log.info("Ignoring table {} as its underlying location {} does not pass allowlist regex {}", dbAndTable, - table.getSd().getLocation(), tableFolderAllowlistRegex); + table.getSd().getLocation(), tableFolderAllowlistRegex.get()); continue; } @@ -304,11 +304,11 @@ protected HiveDataset computeNext() { }; } - protected static boolean shouldAllowTableLocation(Optional regex, Table table) { + protected static boolean shouldAllowTableLocation(Optional regex, Table table) { if (!regex.isPresent()) { return true; } - return Pattern.compile(regex.get()).matcher(table.getSd().getLocation()).matches(); + return regex.get().matcher(table.getSd().getLocation()).matches(); } /** diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java index 25b06201bec..a9805d39540 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinderTest.java @@ -216,9 +216,10 @@ public void testDatasetConfig() throws Exception { } @Test - public void testHiveTableFolderFilter() throws Exception { + public void testHiveTableFolderAllowlistFilter() throws Exception { List dbAndTables = Lists.newArrayList(); dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1")); + // This table is created on /tmp/test HiveMetastoreClientPool pool = getTestPool(dbAndTables); Properties properties = new Properties(); @@ -239,6 +240,24 @@ public void testHiveTableFolderFilter() throws Exception { datasets = Lists.newArrayList(finder.getDatasetsIterator()); Assert.assertEquals(datasets.size(), 0); + + // Test empty filter + properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, ""); + // The table located at /tmp/test should be filtered + properties.put(HiveDatasetFinder.TABLE_FOLDER_ALLOWLIST_FILTER, ""); + + finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); + datasets = Lists.newArrayList(finder.getDatasetsIterator()); + + Assert.assertEquals(datasets.size(), 0); + + // Test no regex config + properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, ""); + + finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool); + datasets = Lists.newArrayList(finder.getDatasetsIterator()); + + Assert.assertEquals(datasets.size(), 0); } private HiveMetastoreClientPool getTestPool(List dbAndTables) throws Exception {