From af81d82b966de63bef75e44637984471cc9221b8 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Wed, 10 Mar 2021 16:46:00 +0100 Subject: [PATCH] [#7] Support inserting and querying arrays --- .circleci/config.yml | 72 ++++++++++----------- README.md | 1 + deps.edn | 1 + script/generate_circleci.clj | 6 +- script/install-clojure | 6 +- script/test | 8 +-- src/pod/babashka/sql.clj | 90 ++++++++++++++++++++++++--- test/pod/babashka/hsqldb_test.clj | 11 +++- test/pod/babashka/postgresql_test.clj | 15 ++++- 9 files changed, 150 insertions(+), 60 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a4defca..389cfac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,9 +20,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -56,8 +56,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java8-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} hsqldb-linux-static: docker: @@ -78,9 +78,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -114,8 +114,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java8-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} hsqldb-mac: macos: {xcode: 12.0.0} @@ -163,8 +163,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java8-20.3.0] + key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} postgresql-linux: docker: @@ -185,9 +185,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -221,8 +221,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} postgresql-linux-static: docker: @@ -243,9 +243,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -279,8 +279,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} postgresql-mac: macos: {xcode: 12.0.0} @@ -328,8 +328,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} oracle-linux: docker: @@ -350,9 +350,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -386,8 +386,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} oracle-linux-static: docker: @@ -408,9 +408,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -444,8 +444,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} oracle-mac: macos: {xcode: 12.0.0} @@ -493,8 +493,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} mssql-linux: docker: @@ -515,9 +515,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -551,8 +551,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} mssql-linux-static: docker: @@ -573,9 +573,9 @@ jobs: name: Install Clojure command: |2- - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh + wget https://download.clojure.org/install/linux-install-1.10.2.796.sh + chmod +x linux-install-1.10.2.796.sh + sudo ./linux-install-1.10.2.796.sh - run: name: Install lsof command: | @@ -609,8 +609,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} mssql-mac: macos: {xcode: 12.0.0} @@ -658,8 +658,8 @@ jobs: command: | .circleci/script/release - save_cache: - key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} paths: [~/.m2, ~/graalvm-ce-java11-20.3.0] + key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: {path: /tmp/release, destination: release} workflows: version: 2 diff --git a/README.md b/README.md index 2f56c60..d8965e2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ the database type, either `hsqldb`, `postgresql`, or `oracle`: - `pod.babashka.`: - `execute!`: similar to `next.jdbc/execute!` + - `execute-one!`: similar to `next.jdbc/execute-one!` - `get-connection`: returns connection serialized using maps with a unique identifier key - `close-connection`: closes a connection returned from `get-connection` - `with-transaction`: similar to `next.jdbc/with-transaction` diff --git a/deps.edn b/deps.edn index daab7d1..0282bd5 100644 --- a/deps.edn +++ b/deps.edn @@ -10,6 +10,7 @@ :sha "cb96e80f6f3d3b307c59cbeb49bb0dcb3a2a780b"} com.opentable.components/otj-pg-embedded {:mvn/version "0.13.3"} babashka/babashka.pods + #_{:local/root "../pods"} {:git/url "https://github.com/babashka/babashka.pods" :sha "a033bccaf0024b279480f2a7ac4b446621629b73"}} :extra-paths ["test"] diff --git a/script/generate_circleci.clj b/script/generate_circleci.clj index bda5f60..de8470c 100755 --- a/script/generate_circleci.clj +++ b/script/generate_circleci.clj @@ -23,9 +23,9 @@ {:restore_cache {:keys ["linux-{{ checksum \"project.clj\" }}-{{ checksum \".circleci/config.yml\" }}"]}} {:run {:name "Install Clojure", :command " -wget https://download.clojure.org/install/linux-install-1.10.1.447.sh -chmod +x linux-install-1.10.1.447.sh -sudo ./linux-install-1.10.1.447.sh"}} +wget https://download.clojure.org/install/linux-install-1.10.2.796.sh +chmod +x linux-install-1.10.2.796.sh +sudo ./linux-install-1.10.2.796.sh"}} {:run {:name "Install lsof", :command "sudo apt-get install lsof\n"}} {:run {:name "Install native dev tools", diff --git a/script/install-clojure b/script/install-clojure index f4781ba..7583a79 100755 --- a/script/install-clojure +++ b/script/install-clojure @@ -3,8 +3,8 @@ install_dir=${1:-/usr/local} mkdir -p "$install_dir" cd /tmp -curl -O -sL https://download.clojure.org/install/clojure-tools-1.10.1.447.tar.gz -tar xzf clojure-tools-1.10.1.447.tar.gz +curl -O -sL https://download.clojure.org/install/clojure-tools-1.10.2.796.tar.gz +tar xzf clojure-tools-1.10.2.796.tar.gz cd clojure-tools clojure_lib_dir="$install_dir/lib/clojure" mkdir -p "$clojure_lib_dir/libexec" @@ -18,6 +18,6 @@ cp clojure "$install_dir/bin" cp clj "$install_dir/bin" cd /tmp -rm -rf clojure-tools-1.10.1.447.tar.gz +rm -rf clojure-tools-1.10.2.796.tar.gz rm -rf clojure-tools echo "Installed clojure to $install_dir/bin" diff --git a/script/test b/script/test index 0a95bdc..bc41cb8 100755 --- a/script/test +++ b/script/test @@ -2,20 +2,20 @@ if [ "$POD_DB_TYPE" = "postgresql" ] then - clojure -A:test -n pod.babashka.postgresql-test + clojure -M:test -n pod.babashka.postgresql-test fi if [ "$POD_DB_TYPE" = "hsqldb" ] then - clojure -A:test -n pod.babashka.hsqldb-test + clojure -M:test -n pod.babashka.hsqldb-test fi if [ "$POD_DB_TYPE" = "oracle" ] then - clojure -A:test -n pod.babashka.oracle-test + clojure -M:test -n pod.babashka.oracle-test fi if [ "$POD_DB_TYPE" = "mssql" ] then - clojure -A:test -n pod.babashka.mssql-test + clojure -M:test -n pod.babashka.mssql-test fi diff --git a/src/pod/babashka/sql.clj b/src/pod/babashka/sql.clj index 2c1a8e1..f7e19bd 100644 --- a/src/pod/babashka/sql.clj +++ b/src/pod/babashka/sql.clj @@ -7,12 +7,19 @@ [cognitect.transit :as transit] [next.jdbc :as jdbc] [next.jdbc.date-time] + [next.jdbc.result-set :as rs] [next.jdbc.sql :as sql] [next.jdbc.transaction :as t] [pod.babashka.sql.features :as features]) - (:import [java.io PushbackInputStream]) + (:import [java.io PushbackInputStream] + [java.sql Array]) (:gen-class)) +(extend-protocol rs/ReadableColumn + Array + (read-column-by-label [^Array v _] (vec (.getArray v))) + (read-column-by-index [^Array v _ _] (vec (.getArray v)))) + (def stdin (PushbackInputStream. System/in)) (defn write [v] @@ -49,14 +56,28 @@ (get @conns conn-id) db-spec)) -(defn execute! [db-spec & args] - (let [conn (->connectable db-spec)] +(defn deserialize [xs] + (if (map? xs) + (if-let [arr (:pod.babashka.sql/array xs)] + (into-array arr) + xs) + xs)) + +(defn -execute! [db-spec & args] + (let [args (walk/postwalk deserialize args) + conn (->connectable db-spec)] (apply jdbc/execute! conn args))) -(defn execute-one! [db-spec & args] - (let [conn (->connectable db-spec)] +(defn -execute-one! [db-spec & args] + (let [args (walk/postwalk deserialize args) + conn (->connectable db-spec)] (apply jdbc/execute-one! conn args))) +(defn -insert-multi! [db-spec table cols rows] + (let [rows (walk/postwalk deserialize rows) + conn (->connectable db-spec)] + (sql/insert-multi! conn table cols rows))) + (defn close-connection [{:keys [::connection]}] (let [[old _new] (swap-vals! conns dissoc connection)] (when-let [conn (get old connection)] @@ -102,15 +123,21 @@ features/mssql? "pod.babashka.mssql" :else (throw (Exception. "Feature flag expected.")))) +(def sql-sql-ns (cond features/postgresql? "pod.babashka.postgresql.sql" + features/hsqldb? "pod.babashka.hsqldb.sql" + features/oracle? "pod.babashka.oracle.sql" + features/mssql? "pod.babashka.mssql.sql" + :else (throw (Exception. "Feature flag expected.")))) + (def lookup - (let [m {'execute! execute! - 'execute-one! execute-one! + (let [m {'-execute! -execute! + '-execute-one! -execute-one! 'get-connection get-connection 'close-connection close-connection 'transaction/begin transaction-begin 'transaction/rollback transaction-rollback 'transaction/commit transaction-commit - 'sql/insert-multi! sql/insert-multi!}] + 'sql/-insert-multi! -insert-multi!}] (zipmap (map (fn [sym] (if-let [ns (namespace sym)] (symbol (str sql-ns "." ns) (name sym)) @@ -123,6 +150,38 @@ slurp (str/replace "pod.babashka.sql" sql-ns))) +(defn replace-sql-ns [s] + (-> s + (str/replace "sqlns" sql-ns) + (str/replace "sql-sql-ns" sql-sql-ns))) + +(def execute-str + (replace-sql-ns (str '(defn execute! [db-spec & args] + (apply sqlns/-execute! db-spec (sqlns/-serialize args)))))) + +(def execute-one-str + (replace-sql-ns (str '(defn execute-one! [db-spec & args] + (apply sqlns/-execute-one! db-spec (sqlns/-serialize args)))))) + +(def insert-multi-str + (-> (str '(defn insert-multi! [db-spec table cols rows] + (sql-sql-ns/-insert-multi! db-spec table cols (sqlns/-serialize rows)))) + replace-sql-ns)) + +(def -wrap-array-str + (pr-str '(defn -wrap-array [x] + (if-let [c (class x)] + (if (.isArray c) + {::array (vec x)} + x) + x)))) + +(def -serialize-str + (replace-sql-ns + (pr-str '(do (require 'clojure.walk) + (defn -serialize [obj] + (clojure.walk/postwalk sqlns/-wrap-array obj)))))) + (def describe-map (walk/postwalk (fn [v] @@ -130,7 +189,16 @@ v)) `{:format :transit+json :namespaces [{:name ~(symbol sql-ns) - :vars [{:name execute!} + :vars [{:name -execute!} + {:name -execute-one!} + {:name -wrap-array + :code ~-wrap-array-str} + {:name -serialize + :code ~-serialize-str} + {:name execute! + :code ~execute-str} + {:name execute-one! + :code ~execute-one-str} {:name get-connection} {:name close-connection} {:name with-transaction @@ -140,7 +208,9 @@ {:name rollback} {:name commit}]} {:name ~(symbol (str sql-ns ".sql")) - :vars [{:name insert-multi!}]}] + :vars [{:name -insert-multi!} + {:name insert-multi! + :code ~insert-multi-str}]}] :opts {:shutdown {}}})) (debug describe-map) diff --git a/test/pod/babashka/hsqldb_test.clj b/test/pod/babashka/hsqldb_test.clj index eab3e84..eaabec7 100644 --- a/test/pod/babashka/hsqldb_test.clj +++ b/test/pod/babashka/hsqldb_test.clj @@ -2,7 +2,6 @@ {:clj-kondo/config '{:lint-as {pod.babashka.hsqldb/with-transaction next.jdbc/with-transaction}}} (:require [babashka.pods :as pods] - [pod.babashka.sql.features :as features] [clojure.test :refer [deftest is testing]])) (pods/load-pod (if (= "native" (System/getenv "POD_TEST_ENV")) @@ -58,4 +57,12 @@ (is (= [#:FOO{:FOO 1} #:FOO{:FOO 2} #:FOO{:FOO 3} #:FOO{:FOO 4} #:FOO{:FOO 5} #:FOO{:FOO 6} #:FOO{:FOO 7}] - (db/execute! db ["select * from foo;"])))))))) + (db/execute! db ["select * from foo;"])))))) + (db/execute! db ["drop schema public cascade;"])) + (let [db "jdbc:hsqldb:mem:testdb;sql.syntax_mys=true"] + (is (db/execute! db ["create table foo ( foo integer array );"])) + (is (db/execute! db ["insert into foo (foo) values (?);" (into-array [1 2 3])])) + (is (= [#:FOO{:FOO [1 2 3]}] + (db/execute! db ["select * from foo"]))) + (is (= #:FOO{:FOO [1 2 3]} + (db/execute-one! db ["select * from foo"]))))) diff --git a/test/pod/babashka/postgresql_test.clj b/test/pod/babashka/postgresql_test.clj index c3d647e..6675afb 100644 --- a/test/pod/babashka/postgresql_test.clj +++ b/test/pod/babashka/postgresql_test.clj @@ -2,7 +2,6 @@ {:clj-kondo/config '{:lint-as {pod.babashka.postgresql/with-transaction next.jdbc/with-transaction}}} (:require [babashka.pods :as pods] - [pod.babashka.sql.features :as features] [clojure.test :refer [deftest is testing]]) (:import [com.opentable.db.postgres.embedded EmbeddedPostgres] [java.util Date])) @@ -13,6 +12,7 @@ "run" "-m" "pod.babashka.sql"])) (require '[pod.babashka.postgresql :as db]) +(require '[pod.babashka.postgresql.sql :as sql]) (require '[pod.babashka.postgresql.transaction :as transaction]) (def port 54322) @@ -76,4 +76,15 @@ (db/execute! x ["insert into foo values (8);"])))) (is (= [#:foo{:foo 1} #:foo{:foo 2} #:foo{:foo 3} #:foo{:foo 4} #:foo{:foo 5} #:foo{:foo 6} #:foo{:foo 7}] - (db/execute! db ["select * from foo;"])))))))) + (db/execute! db ["select * from foo;"])))))) + (testing "inserting an array" + (is (db/execute! db ["create table bar ( bar integer[] );"])) + (is (db/execute! db ["insert into bar values (?);" (into-array [1 2 3])])) + (is (= [#:bar{:bar [1 2 3]}] (db/execute! db ["select * from bar"]))) + (is (db/execute! db ["create table baz ( baz text[] );"])) + (is (db/execute! db ["insert into baz values (?);" (into-array ["foo" "bar"])])) + (is (= [#:baz{:baz ["foo" "bar"]}] (db/execute! db ["select * from baz"]))) + (is (= #:baz{:baz ["foo" "bar"]} (db/execute-one! db ["select * from baz"]))) + (is (= [#:baz{:baz ["a" "b"]} #:baz{:baz ["x" "y"]}] + (sql/insert-multi! db :baz [:baz] [[(into-array ["a" "b"])] + [(into-array ["x" "y"])]]))))))