feat(clickhouse): optimize extract tables with caching

openmeterio · Sep 15, 2023 · 7a8004a · 7a8004a
1 parent a93eceb
commit 7a8004a
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java
@@ -86,7 +86,7 @@ public void updateMapping() {
         LOGGER.debug("Update table mapping.");
 
         // Getting tables from ClickHouse
-        List<Table> tableList = this.chc.extractTablesMapping();
+        List<Table> tableList = this.chc.extractTablesMapping(this.mapping);
         if (tableList.isEmpty()) {
             return;
         }

diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java
@@ -166,9 +166,21 @@ public Table describeTable(String tableName) {
             return null;
         }
     }
+
     public List<Table> extractTablesMapping() {
+        HashMap<String, Table> cache = new HashMap<String, Table>();
+        return extractTablesMapping(cache);
+    }
+
+    public List<Table> extractTablesMapping(Map<String, Table> cache) {
         List<Table> tableList =  new ArrayList<>();
         for (String tableName : showTables() ) {
+            // Read from cache if we already described this table before
+            if (cache.containsKey(tableName)) {
+                tableList.add(cache.get(tableName));
+                continue;
+            }
+
             Table table = describeTable(tableName);
             if (table != null )
                 tableList.add(table);