- preinstall DuckDB magic extension in the docker image

- minor JdbcFactory refactoring

docker/Dockerfile

@@ -163,6 +163,12 @@ COPY firefly/docker/local.xml conf/Catalina/localhost
 #copy all wars, typically there should only be one
 COPY --from=builder /opt/work/${build_dir}/build/dist/*.war ${CATALINA_HOME}/webapps-ref/
 
+# preinstall DuckDB extensions; remember to update version when DuckDB is updated
+ARG DUCKDB_TARGET=v1.1.3/linux_amd64_gcc4
+RUN mkdir -p temp/$DUCKDB_TARGET \
+    && wget -P temp/$DUCKDB_TARGET http://community-extensions.duckdb.org/$DUCKDB_TARGET/magic.duckdb_extension.gz \
+    && gunzip -f temp/$DUCKDB_TARGET/magic.duckdb_extension.gz
+
 # Add permission to files and directories needed for runtime
 # increase max header size to avoid failing on large auth token
 WORKDIR ${CATALINA_HOME}

src/firefly/java/edu/caltech/ipac/firefly/server/cache/EhcacheImpl.java

@@ -29,25 +29,15 @@ public EhcacheImpl(Ehcache cache) {
     }
 
     public void put(CacheKey key, Object value) {
-//        logger.briefDebug("cache pre-put:" + key + " = " + StringUtils.toString(value));
         String keystr = key.getUniqueString();
         if (value == null) {
             cache.remove(keystr);
         } else {
             cache.put(new Element(keystr, value));
         }
-//        logger.briefDebug("cache aft-put:" + key + " = " + StringUtils.toString(value));
     }
 
     public void put(CacheKey key, Object value, int lifespanInSecs) {
-//        logger.briefDebug("cache pre-put:" + key +  " = " + StringUtils.toString(value) +
-//                          " lifespanInSecs:" + lifespanInSecs);
-
-        if (!cache.getCacheConfiguration().isEternal()) {
-            throw new UnsupportedOperationException("Currently, we do not support cached object" +
-                    " with idle time expiry and lifespan expiry at the same time.");
-        }
-
         String keystr = key.getUniqueString();
         if (value == null) {
             cache.remove(keystr);
@@ -56,8 +46,6 @@ public void put(CacheKey key, Object value, int lifespanInSecs) {
             el.setTimeToLive(lifespanInSecs);
             cache.put(el);
         }
-//        logger.briefDebug("cache aft-put:" + key +  " = " + StringUtils.toString(value) +
-//                          " lifespanInSecs:" + lifespanInSecs);
     }
 
     public Object get(CacheKey key) {

src/firefly/java/edu/caltech/ipac/firefly/server/db/DbInstance.java

@@ -109,7 +109,7 @@ public String getDbUrl() {
     public void setPooled(boolean pooled) { isPooled = pooled;}
     public boolean testConn(Connection conn) {
         try (Statement stmt = conn.createStatement()) {
-            stmt.execute("SELECT 1 FROM (VALUES (0))");     // HSQL required FROM clause
+            stmt.execute("SELECT 1 FROM (VALUES (0)) AS dummy");     // HSQL required FROM clause; postgres required alias
             return true;
         } catch (SQLException e) { return false; }
     };

src/firefly/java/edu/caltech/ipac/firefly/server/db/DuckDbAdapter.java

@@ -9,6 +9,7 @@
 import edu.caltech.ipac.firefly.server.db.spring.JdbcFactory;
 import edu.caltech.ipac.firefly.server.query.DataAccessException;
 import edu.caltech.ipac.firefly.server.util.Logger;
+import edu.caltech.ipac.firefly.util.Ref;
 import edu.caltech.ipac.table.DataGroup;
 import edu.caltech.ipac.table.DataType;
 import edu.caltech.ipac.util.AppProperties;
@@ -29,8 +30,11 @@
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static edu.caltech.ipac.firefly.core.Util.Try;
 import static edu.caltech.ipac.firefly.server.db.DuckDbUDF.*;
@@ -44,6 +48,7 @@
 public class DuckDbAdapter extends BaseDbAdapter {
     public static final String NAME = "duckdb";
     public static final String DRIVER = "org.duckdb.DuckDBDriver";
+    public static final String EXT_DIR = AppProperties.getProperty("duckdb.ext.dir", System.getProperty("java.io.tmpdir"));
     public static String maxMemory = AppProperties.getProperty("duckdb.max.memory");        // in GB; 2G, 5.5G, etc
     private static int threadCnt=1;    // min 125mb per thread.  recommend 5gb per thread; we will config 1gb per thread but not more than 4.
 
@@ -87,7 +92,7 @@ public boolean testConn(Connection conn) {
                 } catch (SQLException e) { return false; }
             }
         };
-        db.consumeProps("memory_limit=%s,threads=%d".formatted(maxMemory, threadCnt));
+        db.consumeProps("memory_limit=%s,threads=%d,extension_directory=%s".formatted(maxMemory, threadCnt, EXT_DIR));
         return db;
     }
 
@@ -279,34 +284,40 @@ public static String replaceLike(String input) {
     public record MimeDesc(String mime, String desc) {}
     @Nonnull
     public static MimeDesc getMimeType(File inFile) {
-        String tmpDir = System.getProperty("java.io.tmpdir");
-        try (Connection conn = new DuckDbAdapter().getJdbcTmpl().getDataSource().getConnection();
-             Statement stmt = conn.createStatement();
-             ResultSet extrs = stmt.executeQuery("SET extension_directory = '%s'; SELECT installed, loaded FROM duckdb_extensions() WHERE extension_name = 'magic'".formatted(tmpDir))) {
-
-            boolean installed = false;
-            boolean loaded = false;
-            if (extrs.next()) {
-                installed = extrs.getBoolean("installed");
-                loaded = extrs.getBoolean("loaded");
+        try(JdbcFactory.SharedDS ds = JdbcFactory.getSharedDS(new DuckDbAdapter().createDbInstance())) {
+            loadExtension(ds, "magic");
+            Map<String, Object> rs = ds.getJdbc().queryForMap("SELECT file, magic_mime(file) AS mime, magic_type(file) AS desc FROM glob('%s')".formatted(inFile.getAbsolutePath()));
+            if (!rs.isEmpty()) {
+                return new MimeDesc(String.valueOf(rs.get("mime")), String.valueOf(rs.get("desc")));
             }
-            // Install and load the extension if not already done
-            if (!installed) {
-                stmt.executeUpdate("SET extension_directory = '%s'; INSTALL magic FROM community".formatted(tmpDir));
-            }
-            if (!loaded) {
-                stmt.executeUpdate("SET extension_directory = '%s'; LOAD magic".formatted(tmpDir));
-            }
-            try (ResultSet rs = stmt.executeQuery("SELECT file, magic_mime(file) AS mime, magic_type(file) AS desc FROM glob('%s')".formatted(inFile.getAbsolutePath()))) {
-                if (rs.next()) {
-                    return new MimeDesc(rs.getString("mime"), rs.getString("desc"));
-                }
-            }
-
-        } catch (SQLException ex) {
+        } catch (Exception ex) {
             Logger.getLogger().error(ex, "Failed to detect mime type");
         }
         return new MimeDesc("application/x-unknown", "unknown");
     }
 
-}
+    /**
+     * Loads the specified extension in DuckDB.
+     *
+     * @param ds the shared data source
+     * @param name the name of the extension to load
+     */
+    static void loadExtension(JdbcFactory.SharedDS ds, String name) {
+        SimpleJdbcTemplate jdbc = ds.getJdbc();
+        Ref<Boolean> installed = new Ref<>(false);
+        Ref<Boolean> loaded = new Ref<>(false);
+        jdbc.query("SELECT installed, loaded FROM duckdb_extensions() WHERE extension_name = '%s'".formatted(name), (rs, idx) -> {
+            installed.set(rs.getBoolean("installed"));
+            loaded.set(rs.getBoolean("loaded"));
+            return null;
+        });
+        // Install and load the extension if not already done
+        if (!installed.get()) {
+            jdbc.update("INSTALL %s FROM community".formatted(name));
+        }
+        if (!loaded.get()) {
+            jdbc.update("LOAD %s".formatted(name));
+        }
+    }
+
+}

src/firefly/java/edu/caltech/ipac/firefly/server/db/spring/JdbcFactory.java

@@ -73,43 +73,16 @@ public static SimpleJdbcTemplate getSimpleTemplate(DbInstance dbInstance) {
     }
 
     /**
-     * return a JdbcTemplate with a single underlying connection.
-     * this allows a user to perform multiple tasks on one connection.
-     * this implementation is not thread-safe.
-     * @param dbInstance
-     * @return
-     */
-    public static JdbcTemplate getStatefulTemplate(DbInstance dbInstance) {
-        DataSource ds = getSingleConnectionDS(dbInstance);
-        return ds == null ? null : new JdbcTemplate(ds);
-    }
-
-    /**
-     * return a simple JdbcTemplate with a single underlying connection.
-     * this allow a user to perform multiple tasks on one connection.
-     * this implementation is not thread-safe.
-     * @param dbInstance
-     * @return
+     * Returns a DataSource wrapper that ensures the underlying connection is not closed automatically.
+     * The connection will remain open until explicitly closed by the caller,
+     * allowing multiple tasks to be executed on the same connection.
+     * Note: This implementation is not thread-safe.
+     *
+     * @param dbInstance the database to connect to
+     * @return a {@link SharedDS} instance for the specified database instance
      */
-    public static SimpleJdbcTemplate getStatefulSimpleTemplate(DbInstance dbInstance) {
-        DataSource ds = getSingleConnectionDS(dbInstance);
-        return ds == null ? null : new SimpleJdbcTemplate(ds);
-    }
-
-    /**
-     * return a DataSource with a single underlying connection.
-     * this allow a user to perform multiple tasks on one connection.
-     * this implementation is not thread-safe.
-     * @param dbInstance
-     * @return
-     */
-    public static DataSource getSingleConnectionDS(DbInstance dbInstance) {
-        try {
-            return new SingleConnectionDataSource(getDataSource(dbInstance).getConnection(), false);
-        } catch (SQLException e) {
-            logger.error(e);
-        }
-        return null;
+    public static SharedDS getSharedDS(DbInstance dbInstance) {
+        return new SharedDS(dbInstance);
     }
 
     public static DataSource getDataSource(DbInstance dbInstance) {
@@ -179,4 +152,28 @@ private Properties addProps(Properties props) {
         }
 
     }
+
+    public static class SharedDS implements AutoCloseable {
+        DataSource ds;
+        public SharedDS(DbInstance di) {
+            try {
+                ds = new SingleConnectionDataSource(getDataSource(di).getConnection(), false);
+            } catch (Exception e) {
+                logger.error(e);
+                throw new IllegalArgumentException("Failed to get DataSource");
+            }
+        }
+        public void close() throws Exception {
+            Try.it(() -> ds.getConnection().close());
+        }
+        public DataSource get() {
+            return ds;
+        }
+        public SimpleJdbcTemplate getJdbc() {
+            return new SimpleJdbcTemplate(ds);
+        }
+        public JdbcTemplate getTmpl() {
+            return new JdbcTemplate(ds);
+        }
+    }
 }

src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java

@@ -193,6 +193,7 @@ protected void createDbFromRequest(TableServerRequest treq, DbAdapter dbAdapter)
                 }
             }
         } catch (Exception e) {
+            logger.error(e);
             dbAdapter.close(true);
             throw dbAdapter.handleSqlExp("", e);
         }

src/firefly/test/edu/caltech/ipac/firefly/ConfigTest.java

@@ -118,13 +118,13 @@ public static void setupServerContext(RequestAgent requestAgent, String contextN
         webappConfigPath = webappConfigPath == null ? Paths.get("build/%s/war/WEB-INF/config".formatted(contextName)).toAbsolutePath().toString() : webappConfigPath;
 
         AppProperties.setProperty("work.directory", Paths.get("build").toAbsolutePath().toString());
-        Path buildConfg = Paths.get("build/firefly/war/WEB-INF/config");
+        Path buildConfg = Paths.get(webappConfigPath);
+        System.setProperty("java.io.tmpdir", "build/%s/tmp".formatted(contextName));
 
         copyWithSub(Paths.get("./config/ehcache.xml"), buildConfg, "app-name", contextName);
         copy(Paths.get("config/test/app-test.prop"), buildConfg);
         copy(Paths.get("config/ignore_sizeof.txt"), buildConfg);
 
-        webappConfigPath = webappConfigPath != null ? webappConfigPath : buildConfg.toAbsolutePath().toString();
         requestAgent = requestAgent == null ? new RequestAgent(null, "localhost", "/test", "localhost:8080/", "127.0. 0.1", UUID.randomUUID().toString(), contextPath): requestAgent;
 
         ServerContext.getRequestOwner().setRequestAgent(requestAgent);