Use the Trino client instead of inheriting from Presto

cube-js · Nov 13, 2024 · 6fc1639 · 6fc1639
1 parent 5593a2a
commit 6fc1639
Show file tree

Hide file tree

Showing 11 changed files with 340 additions and 56 deletions.
diff --git a/.github/actions/smoke.sh b/.github/actions/smoke.sh
@@ -45,14 +45,14 @@ echo "::endgroup::"
 #yarn lerna run --concurrency 1 --stream --no-prefix smoke:prestodb
 #echo "::endgroup::"
 
-echo "::group::Trino"
-yarn lerna run --concurrency 1 --stream --no-prefix smoke:trino
-echo "::endgroup::"
+#echo "::group::Trino"
+#yarn lerna run --concurrency 1 --stream --no-prefix smoke:trino
+#echo "::endgroup::"
 
 echo "::group::MS SQL"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mssql
 echo "::endgroup::"
 
 echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
-echo "::endgroup::"
+echo "::endgroup::"
diff --git a/docs/pages/product/configuration/data-sources/trino.mdx b/docs/pages/product/configuration/data-sources/trino.mdx
@@ -22,7 +22,7 @@ CUBEJS_DB_TYPE=trino
 CUBEJS_DB_HOST=my.trino.host
 CUBEJS_DB_USER=trino_user
 CUBEJS_DB_PASS=**********
-CUBEJS_DB_PRESTO_CATALOG=my_trino_catalog
+CUBEJS_DB_TRINO_CATALOG=my_trino_catalog
 CUBEJS_DB_SCHEMA=my_trino_schema
 ```
 
@@ -34,9 +34,8 @@ CUBEJS_DB_SCHEMA=my_trino_schema
 | `CUBEJS_DB_PORT`           | The port for the database connection                                                | A valid port number                           |    ❌    |
 | `CUBEJS_DB_USER`           | The username used to connect to the database                                        | A valid database username                     |    ✅    |
 | `CUBEJS_DB_PASS`           | The password used to connect to the database                                        | A valid database password                     |    ✅    |
-| `CUBEJS_DB_PRESTO_CATALOG` | The catalog within Presto to connect to                                             | A valid catalog name within a Presto database |    ✅    |
+| `CUBEJS_DB_TRINO_CATALOG`  | The catalog within Presto to connect to                                             | A valid catalog name within a Presto database |    ✅    |
 | `CUBEJS_DB_SCHEMA`         | The schema within the database to connect to                                        | A valid schema name within a Presto database  |    ✅    |
-| `CUBEJS_DB_SSL`            | If `true`, enables SSL encryption for database connections from Cube                | `true`, `false`                               |    ❌    |
 | `CUBEJS_CONCURRENCY`       | The number of concurrent connections each queue has to the database. Default is `2` | A valid number                                |    ❌    |
 | `CUBEJS_DB_MAX_POOL`       | The maximum number of concurrent database connections to pool. Default is `8`       | A valid number                                |    ❌    |
 

diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts
@@ -66,8 +66,7 @@ export function assertDataSource(dataSource = 'default'): string {
     return dataSource;
   } else {
     throw new Error(
-      `The ${
-        dataSource
+      `The ${dataSource
       } data source is missing in the declared CUBEJS_DATASOURCES.`
     );
   }
@@ -90,10 +89,8 @@ export function keyByDataSource(origin: string, dataSource?: string): string {
       return `CUBEJS_DS_${dataSource.toUpperCase()}_${s[1]}`;
     } else {
       throw new Error(
-        `The ${
-          origin
-        } environment variable can not be converted for the ${
-          dataSource
+        `The ${origin
+        } environment variable can not be converted for the ${dataSource
         } data source.`
       );
     }
@@ -237,8 +234,7 @@ const variables: Record<string, (...args: any) => any> = {
       return false;
     } else {
       throw new TypeError(
-        `The ${
-          keyByDataSource('CUBEJS_DB_SSL', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_SSL', dataSource)
         } must be either 'true' or 'false'.`
       );
     }
@@ -261,8 +257,7 @@ const variables: Record<string, (...args: any) => any> = {
       return false;
     } else {
       throw new TypeError(
-        `The ${
-          keyByDataSource('CUBEJS_DB_SSL_REJECT_UNAUTHORIZED', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_SSL_REJECT_UNAUTHORIZED', dataSource)
         } must be either 'true' or 'false'.`
       );
     }
@@ -353,8 +348,7 @@ const variables: Record<string, (...args: any) => any> = {
   }) => (
     process.env[keyByDataSource('CUBEJS_DB_PORT', dataSource)]
       ? parseInt(
-        `${
-          process.env[keyByDataSource('CUBEJS_DB_PORT', dataSource)]
+        `${process.env[keyByDataSource('CUBEJS_DB_PORT', dataSource)]
         }`,
         10,
       )
@@ -409,8 +403,7 @@ const variables: Record<string, (...args: any) => any> = {
     ];
     if (required && !val) {
       throw new Error(
-        `The ${
-          keyByDataSource('CUBEJS_DB_NAME', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_NAME', dataSource)
         } is required and missing.`
       );
     }
@@ -429,19 +422,16 @@ const variables: Record<string, (...args: any) => any> = {
     required?: boolean,
   }) => {
     console.warn(
-      `The ${
-        keyByDataSource('CUBEJS_DB_SCHEMA', dataSource)
-      } is deprecated. Please, use the ${
-        keyByDataSource('CUBEJS_DB_NAME', dataSource)
+      `The ${keyByDataSource('CUBEJS_DB_SCHEMA', dataSource)
+      } is deprecated. Please, use the ${keyByDataSource('CUBEJS_DB_NAME', dataSource)
       } instead.`
     );
     const val = process.env[
       keyByDataSource('CUBEJS_DB_SCHEMA', dataSource)
     ];
     if (required && !val) {
       throw new Error(
-        `The ${
-          keyByDataSource('CUBEJS_DB_SCHEMA', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_SCHEMA', dataSource)
         } is required and missing.`
       );
     }
@@ -460,19 +450,16 @@ const variables: Record<string, (...args: any) => any> = {
     required?: boolean,
   }) => {
     console.warn(
-      `The ${
-        keyByDataSource('CUBEJS_DATABASE', dataSource)
-      } is deprecated. Please, use the ${
-        keyByDataSource('CUBEJS_DB_NAME', dataSource)
+      `The ${keyByDataSource('CUBEJS_DATABASE', dataSource)
+      } is deprecated. Please, use the ${keyByDataSource('CUBEJS_DB_NAME', dataSource)
       } instead.`
     );
     const val = process.env[
       keyByDataSource('CUBEJS_DATABASE', dataSource)
     ];
     if (required && !val) {
       throw new Error(
-        `The ${
-          keyByDataSource('CUBEJS_DATABASE', dataSource)
+        `The ${keyByDataSource('CUBEJS_DATABASE', dataSource)
         } is required and missing.`
       );
     }
@@ -489,10 +476,9 @@ const variables: Record<string, (...args: any) => any> = {
   }) => (
     process.env[keyByDataSource('CUBEJS_DB_MAX_POOL', dataSource)]
       ? parseInt(
-        `${
-          process.env[
-            keyByDataSource('CUBEJS_DB_MAX_POOL', dataSource)
-          ]
+        `${process.env[
+          keyByDataSource('CUBEJS_DB_MAX_POOL', dataSource)
+        ]
         }`,
         10,
       )
@@ -684,8 +670,7 @@ const variables: Record<string, (...args: any) => any> = {
       supported.indexOf(<'s3' | 'gcp' | 'azure'>val) === -1
     ) {
       throw new TypeError(
-        `The ${
-          keyByDataSource('CUBEJS_DB_EXPORT_BUCKET_TYPE', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_EXPORT_BUCKET_TYPE', dataSource)
         } must be one of the [${supported.join(', ')}].`
       );
     }
@@ -878,8 +863,7 @@ const variables: Record<string, (...args: any) => any> = {
     ];
     if (!val) {
       throw new Error(
-        `The ${
-          keyByDataSource('CUBEJS_DB_DATABRICKS_URL', dataSource)
+        `The ${keyByDataSource('CUBEJS_DB_DATABRICKS_URL', dataSource)
         } is required and missing.`
       );
     }
@@ -1387,11 +1371,10 @@ const variables: Record<string, (...args: any) => any> = {
         return false;
       } else {
         throw new TypeError(
-          `The ${
-            keyByDataSource(
-              'CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE',
-              dataSource,
-            )
+          `The ${keyByDataSource(
+            'CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE',
+            dataSource,
+          )
           } must be either 'true' or 'false'.`
         );
       }
@@ -1626,6 +1609,17 @@ const variables: Record<string, (...args: any) => any> = {
     ]
   ),
 
+  /**
+   * Trino catalog.
+   */
+  trinoCatalog: ({
+    dataSource,
+  }: { dataSource: string }) => (
+    process.env[
+      keyByDataSource('CUBEJS_DB_TRINO_CATALOG', dataSource)
+    ]
+  ),
+
   /** ****************************************************************
    * Cube Store Driver                                               *
    ***************************************************************** */

diff --git a/packages/cubejs-backend-shared/test/db_env_multi.test.ts b/packages/cubejs-backend-shared/test/db_env_multi.test.ts
@@ -2189,4 +2189,33 @@ describe('Multiple datasources', () => {
       'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
     );
   });
+
+  test('getEnv("trinoCatalog")', () => {
+    process.env.CUBEJS_DB_TRINO_CATALOG = 'default1';
+    process.env.CUBEJS_DS_POSTGRES_DB_TRINO_CATALOG = 'postgres1';
+    process.env.CUBEJS_DS_WRONG_DB_TRINO_CATALOG = 'wrong1';
+    expect(getEnv('trinoCatalog', { dataSource: 'default' })).toEqual('default1');
+    expect(getEnv('trinoCatalog', { dataSource: 'postgres' })).toEqual('postgres1');
+    expect(() => getEnv('trinoCatalog', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    process.env.CUBEJS_DB_TRINO_CATALOG = 'default2';
+    process.env.CUBEJS_DS_POSTGRES_DB_TRINO_CATALOG = 'postgres2';
+    process.env.CUBEJS_DS_WRONG_DB_TRINO_CATALOG = 'wrong2';
+    expect(getEnv('trinoCatalog', { dataSource: 'default' })).toEqual('default2');
+    expect(getEnv('trinoCatalog', { dataSource: 'postgres' })).toEqual('postgres2');
+    expect(() => getEnv('trinoCatalog', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    delete process.env.CUBEJS_DB_TRINO_CATALOG;
+    delete process.env.CUBEJS_DS_POSTGRES_DB_TRINO_CATALOG;
+    delete process.env.CUBEJS_DS_WRONG_DB_TRINO_CATALOG;
+    expect(getEnv('trinoCatalog', { dataSource: 'default' })).toBeUndefined();
+    expect(getEnv('trinoCatalog', { dataSource: 'postgres' })).toBeUndefined();
+    expect(() => getEnv('trinoCatalog', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+  });
 });
diff --git a/packages/cubejs-testing/test/smoke-trino.test.ts b/packages/cubejs-testing/test/smoke-trino.test.ts
@@ -24,7 +24,7 @@ describe('trino', () => {
 
         CUBEJS_DB_HOST: db.getHost(),
         CUBEJS_DB_PORT: `${db.getMappedPort(8080)}`,
-        CUBEJS_DB_PRESTO_CATALOG: 'memory',
+        CUBEJS_DB_TRINO_CATALOG: 'memory',
         CUBEJS_DB_USER: 'test',
 
         ...DEFAULT_CONFIG,

diff --git a/packages/cubejs-trino-driver/config/catalog/postgresql.properties b/packages/cubejs-trino-driver/config/catalog/postgresql.properties
@@ -0,0 +1,4 @@
+connector.name=postgresql
+connection-url=jdbc:postgresql://postgres:5432/trino
+connection-user=trino
+connection-password=9bee2c3975024292eff5829526722ac2
diff --git a/packages/cubejs-trino-driver/docker-compose.yml b/packages/cubejs-trino-driver/docker-compose.yml
@@ -0,0 +1,13 @@
+version: '2.2'
+
+services:
+  coordinator:
+    image: trinodb/trino
+    ports:
+      - "8080:8080"
+    container_name: "coordinator"
+    healthcheck:
+      test: "trino --execute 'SELECT 1' || exit 1"
+      interval: 10s
+      timeout: 5s
+      retries: 5
diff --git a/packages/cubejs-trino-driver/package.json b/packages/cubejs-trino-driver/package.json
@@ -21,6 +21,8 @@
     "build": "rm -rf dist && npm run tsc",
     "tsc": "tsc",
     "watch": "tsc -w",
+    "integration": "npm run integration:trino",
+    "integration:trino": "jest --verbose --config=jest.config.js dist/test",
     "lint": "eslint src/* --ext .ts",
     "lint:fix": "eslint --fix src/* --ext .ts"
   },
@@ -29,15 +31,23 @@
     "@cubejs-backend/prestodb-driver": "1.1.4",
     "@cubejs-backend/schema-compiler": "1.1.4",
     "@cubejs-backend/shared": "1.1.4",
-    "presto-client": "^0.12.2",
+    "trino-client": "^0.2.3",
     "sqlstring": "^2.3.1"
   },
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public"
   },
   "devDependencies": {
-    "@cubejs-backend/linter": "^1.0.0"
+    "@cubejs-backend/linter": "^1.0.0",
+    "@types/jest": "^27",
+    "jest": "^27",
+    "babel-jest": "^27",
+    "testcontainers": "^10.10.4",
+    "typescript": "~5.2.2"
+  },
+  "jest": {
+    "testEnvironment": "node"
   },
   "eslintConfig": {
     "extends": "../cubejs-linter"