diff --git a/firebase-vscode/package.json b/firebase-vscode/package.json index c3521fb3796..e0e557bdff3 100644 --- a/firebase-vscode/package.json +++ b/firebase-vscode/package.json @@ -14,7 +14,8 @@ "Other" ], "extensionDependencies": [ - "graphql.vscode-graphql-syntax" + "graphql.vscode-graphql-syntax", + "redhat.vscode-yaml" ], "activationEvents": [ "onStartupFinished", @@ -110,7 +111,7 @@ "mono-firebase": { "description": "Firebase icon", "default": { - "fontPath": "./resources/Monicons.woff", + "fontPath": "./resources/monicons.woff", "fontCharacter": "\\F101" } }, @@ -180,11 +181,11 @@ }, { "fileMatch": "dataconnect.yaml", - "url": "./schema/dataconnect-yaml.json" + "url": "./dist/schema/dataconnect-yaml.json" }, { "fileMatch": "connector.yaml", - "url": "./schema/connector-yaml.json" + "url": "./dist/schema/connector-yaml.json" } ] }, diff --git a/firebase-vscode/resources/Monicons.woff b/firebase-vscode/resources/Monicons.woff index e6735ccb638..815694ade6b 100644 Binary files a/firebase-vscode/resources/Monicons.woff and b/firebase-vscode/resources/Monicons.woff differ diff --git a/firebase-vscode/resources/firebase_logo.png b/firebase-vscode/resources/firebase_logo.png index e3e46f77939..c5e09f683f3 100644 Binary files a/firebase-vscode/resources/firebase_logo.png and b/firebase-vscode/resources/firebase_logo.png differ diff --git a/firebase-vscode/src/cli.ts b/firebase-vscode/src/cli.ts index 61bb4810c5c..b4565e7e266 100644 --- a/firebase-vscode/src/cli.ts +++ b/firebase-vscode/src/cli.ts @@ -60,19 +60,10 @@ async function getServiceAccount() { if (e.original?.message) { errorMessage += ` (original: ${e.original.message})`; } - if (process.env.MONOSPACE_ENV) { - // If it can't find a service account in Monospace, that's a blocking - // error and we should throw. - throw new Error( - `Unable to find service account. ` + `requireAuthError: ${errorMessage}` - ); - } else { - // In other environments, it is common to not find a service account. - pluginLogger.debug( - `No service account found (this may be normal), ` + - `requireAuth error output: ${errorMessage}` - ); - } + pluginLogger.debug( + `No service account found (this may be normal), ` + + `requireAuth error output: ${errorMessage}` + ); return null; } if (process.env.WORKSPACE_SERVICE_ACCOUNT_EMAIL) { diff --git a/firebase-vscode/src/core/project.ts b/firebase-vscode/src/core/project.ts index 6d9f65977e5..e218594d40f 100644 --- a/firebase-vscode/src/core/project.ts +++ b/firebase-vscode/src/core/project.ts @@ -1,4 +1,4 @@ -import vscode, { Disposable, ExtensionContext, QuickPickItem } from "vscode"; +import vscode, { Disposable } from "vscode"; import { ExtensionBrokerImpl } from "../extension-broker"; import { computed, effect } from "@preact/signals-react"; import { firebaseRC, updateFirebaseRCProject } from "./config"; @@ -6,7 +6,6 @@ import { FirebaseProjectMetadata } from "../types/project"; import { currentUser, isServiceAccount } from "./user"; import { listProjects } from "../cli"; import { pluginLogger } from "../logger-wrapper"; -import { currentOptions } from "../options"; import { globalSignal } from "../utils/globals"; import { firstWhereDefined } from "../utils/signal"; diff --git a/src/dataconnect/client.ts b/src/dataconnect/client.ts index 1052bd5319c..93afd4cae02 100644 --- a/src/dataconnect/client.ts +++ b/src/dataconnect/client.ts @@ -80,8 +80,9 @@ export async function deleteService( locationId: string, serviceId: string, ): Promise { + // NOTE(fredzqm): Don't force delete yet. Backend would leave orphaned resources. const op = await dataconnectClient().delete( - `projects/${projectId}/locations/${locationId}/services/${serviceId}?force=true`, + `projects/${projectId}/locations/${locationId}/services/${serviceId}`, ); const pollRes = await operationPoller.pollOperation({ apiOrigin: dataconnectOrigin(), diff --git a/src/dataconnect/errors.ts b/src/dataconnect/errors.ts index 5f6f9666580..f14a9b8c5b5 100644 --- a/src/dataconnect/errors.ts +++ b/src/dataconnect/errors.ts @@ -5,28 +5,27 @@ const PRECONDITION_ERROR_TYPESTRING = "type.googleapis.com/google.rpc.Preconditi const INCOMPATIBLE_CONNECTOR_TYPE = "INCOMPATIBLE_CONNECTOR"; export function getIncompatibleSchemaError(err: any): IncompatibleSqlSchemaError | undefined { - const original = err.context?.body?.error || err.orignal; - if (!original) { - // If we can't get the original, rethrow so we don't cover up the original error. - throw err; + const incompatibles = errorDetails(err, INCOMPATIBLE_SCHEMA_ERROR_TYPESTRING); + if (incompatibles.length === 0) { + return undefined; } - const details: any[] = original.details; - const incompatibles = details.filter((d) => - d["@type"]?.includes(INCOMPATIBLE_SCHEMA_ERROR_TYPESTRING), - ); // Should never get multiple incompatible schema errors - return incompatibles[0]; + const incompatible = incompatibles[0]; + // Extract the violation type from the precondition error detail. + const preconditionErrs = errorDetails(err, PRECONDITION_ERROR_TYPESTRING); + const violationTypes = (incompatible.violationType = preconditionErrs + .flatMap((preCondErr) => preCondErr.violations) + .flatMap((viol) => viol.type) + .filter((type) => type === "INACCESSIBLE_SCHEMA" || type === "INCOMPATIBLE_SCHEMA")); + incompatible.violationType = violationTypes[0]; + return incompatible; } // Note - the backend just includes file name, not the name of the connector resource in the GQLerror extensions. // so we don't use this yet. Ideally, we'd just include connector name in the extensions. export function getInvalidConnectors(err: any): string[] { + const preconditionErrs = errorDetails(err, PRECONDITION_ERROR_TYPESTRING); const invalidConns: string[] = []; - const original = err.context?.body?.error || err?.orignal; - const details: any[] = original?.details; - const preconditionErrs = details?.filter((d) => - d["@type"]?.includes(PRECONDITION_ERROR_TYPESTRING), - ); for (const preconditionErr of preconditionErrs) { const incompatibleConnViolation = preconditionErr?.violations?.filter( (v: { type: string }) => v.type === INCOMPATIBLE_CONNECTOR_TYPE, @@ -36,3 +35,9 @@ export function getInvalidConnectors(err: any): string[] { } return invalidConns; } + +function errorDetails(err: any, ofType: string): any[] { + const original = err.context?.body?.error || err?.original; + const details: any[] = original?.details; + return details?.filter((d) => d["@type"]?.includes(ofType)) || []; +} diff --git a/src/dataconnect/provisionCloudSql.ts b/src/dataconnect/provisionCloudSql.ts index 18d7a439f6e..a8e1ec14d20 100755 --- a/src/dataconnect/provisionCloudSql.ts +++ b/src/dataconnect/provisionCloudSql.ts @@ -27,13 +27,14 @@ export async function provisionCloudSql(args: { const existingInstance = await cloudSqlAdminClient.getInstance(projectId, instanceId); silent || utils.logLabeledBullet("dataconnect", `Found existing instance ${instanceId}.`); connectionName = existingInstance?.connectionName || ""; - if (!checkInstanceConfig(existingInstance, enableGoogleMlIntegration)) { - // TODO: Return message from checkInstanceConfig to explain exactly what changes are made + const why = getUpdateReason(existingInstance, enableGoogleMlIntegration); + if (why) { silent || utils.logLabeledBullet( "dataconnect", `Instance ${instanceId} settings not compatible with Firebase Data Connect. ` + - `Updating instance to enable Cloud IAM authentication and public IP. This may take a few minutes...`, + `Updating instance. This may take a few minutes...` + + why, ); await promiseWithSpinner( () => @@ -77,11 +78,21 @@ export async function provisionCloudSql(args: { try { await cloudSqlAdminClient.getDatabase(projectId, instanceId, databaseId); silent || utils.logLabeledBullet("dataconnect", `Found existing database ${databaseId}.`); - } catch (err) { - silent || - utils.logLabeledBullet("dataconnect", `Database ${databaseId} not found, creating it now...`); - await cloudSqlAdminClient.createDatabase(projectId, instanceId, databaseId); - silent || utils.logLabeledBullet("dataconnect", `Database ${databaseId} created.`); + } catch (err: any) { + if (err.status === 404) { + // Create the database if not found. + silent || + utils.logLabeledBullet( + "dataconnect", + `Database ${databaseId} not found, creating it now...`, + ); + await cloudSqlAdminClient.createDatabase(projectId, instanceId, databaseId); + silent || utils.logLabeledBullet("dataconnect", `Database ${databaseId} created.`); + } else { + // Skip it if the database is not accessible. + // Possible that the CSQL instance is in the middle of something. + silent || utils.logLabeledWarning("dataconnect", `Database ${databaseId} is not accessible.`); + } } if (enableGoogleMlIntegration) { await grantRolesToCloudSqlServiceAccount(projectId, instanceId, [GOOGLE_ML_INTEGRATION_ROLE]); @@ -92,26 +103,24 @@ export async function provisionCloudSql(args: { /** * Validate that existing CloudSQL instances have the necessary settings. */ -export function checkInstanceConfig( - instance: Instance, - requireGoogleMlIntegration: boolean, -): boolean { +export function getUpdateReason(instance: Instance, requireGoogleMlIntegration: boolean): string { + let reason = ""; const settings = instance.settings; // CloudSQL instances must have public IP enabled to be used with Firebase Data Connect. if (!settings.ipConfiguration?.ipv4Enabled) { - return false; + reason += "\n - to enable public IP."; } if (requireGoogleMlIntegration) { if (!settings.enableGoogleMlIntegration) { - return false; + reason += "\n - to enable Google ML integration."; } if ( !settings.databaseFlags?.some( (f) => f.name === "cloudsql.enable_google_ml_integration" && f.value === "on", ) ) { - return false; + reason += "\n - to enable Google ML integration database flag."; } } @@ -120,6 +129,9 @@ export function checkInstanceConfig( settings.databaseFlags?.some( (f) => f.name === "cloudsql.iam_authentication" && f.value === "on", ) ?? false; + if (!isIamEnabled) { + reason += "\n - to enable IAM authentication database flag."; + } - return isIamEnabled; + return reason; } diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index 385dc2eb9d4..cf49c33a092 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -10,26 +10,40 @@ import { Schema } from "./types"; import { Options } from "../options"; import { FirebaseError } from "../error"; import { needProjectId } from "../projectUtils"; -import { logLabeledWarning, logLabeledSuccess } from "../utils"; +import { logLabeledBullet, logLabeledWarning, logLabeledSuccess } from "../utils"; import * as errors from "./errors"; export async function diffSchema(schema: Schema): Promise { const { serviceName, instanceName, databaseId } = getIdentifiers(schema); - await ensureServiceIsConnectedToCloudSql(serviceName, instanceName, databaseId); + await ensureServiceIsConnectedToCloudSql( + serviceName, + instanceName, + databaseId, + /* linkIfNotConnected=*/ false, + ); try { await upsertSchema(schema, /** validateOnly=*/ true); + logLabeledSuccess("dataconnect", `Database schema is up to date.`); } catch (err: any) { + if (err.status !== 400) { + throw err; + } const invalidConnectors = errors.getInvalidConnectors(err); + const incompatible = errors.getIncompatibleSchemaError(err); + if (!incompatible && !invalidConnectors.length) { + // If we got a different type of error, throw it + throw err; + } + + // Display failed precondition errors nicely. if (invalidConnectors.length) { displayInvalidConnectors(invalidConnectors); } - const incompatible = errors.getIncompatibleSchemaError(err); if (incompatible) { displaySchemaChanges(incompatible); return incompatible.diffs; } } - logLabeledSuccess("dataconnect", `Database schema is up to date.`); return []; } @@ -42,17 +56,27 @@ export async function migrateSchema(args: { const { options, schema, allowNonInteractiveMigration, validateOnly } = args; const { serviceName, instanceId, instanceName, databaseId } = getIdentifiers(schema); - await ensureServiceIsConnectedToCloudSql(serviceName, instanceName, databaseId); + await ensureServiceIsConnectedToCloudSql( + serviceName, + instanceName, + databaseId, + /* linkIfNotConnected=*/ true, + ); try { await upsertSchema(schema, validateOnly); logger.debug(`Database schema was up to date for ${instanceId}:${databaseId}`); } catch (err: any) { + if (err.status !== 400) { + throw err; + } + // Parse and handle failed precondition errors, then retry. const incompatible = errors.getIncompatibleSchemaError(err); const invalidConnectors = errors.getInvalidConnectors(err); if (!incompatible && !invalidConnectors.length) { // If we got a different type of error, throw it throw err; } + const shouldDeleteInvalidConnectors = await promptForInvalidConnectorError( options, invalidConnectors, @@ -61,7 +85,7 @@ export async function migrateSchema(args: { if (!shouldDeleteInvalidConnectors && invalidConnectors.length) { const cmd = suggestedCommand(serviceName, invalidConnectors); throw new FirebaseError( - `Command aborted. Try deploying compatible connectors first with ${clc.bold(cmd)}`, + `Command aborted. Try deploying those connectors first with ${clc.bold(cmd)}`, ); } const migrationMode = incompatible @@ -266,44 +290,61 @@ function displayInvalidConnectors(invalidConnectors: string[]) { // If a service has never had a schema with schemaValidation=strict // (ie when users create a service in console), -// the backend will not have the necesary permissions to check cSQL for differences. +// the backend will not have the necessary permissions to check cSQL for differences. // We fix this by upserting the currently deployed schema with schemaValidation=strict, async function ensureServiceIsConnectedToCloudSql( serviceName: string, instanceId: string, databaseId: string, + linkIfNotConnected: boolean, ) { let currentSchema: Schema; try { currentSchema = await getSchema(serviceName); } catch (err: any) { - if (err.status === 404) { - // If no schema has been deployed yet, deploy an empty one to get connectivity. - currentSchema = { - name: `${serviceName}/schemas/${SCHEMA_ID}`, - source: { - files: [], - }, - primaryDatasource: { - postgresql: { - database: databaseId, - cloudSql: { - instance: instanceId, - }, - }, - }, - }; - } else { + if (err.status !== 404) { throw err; } + if (!linkIfNotConnected) { + logLabeledWarning("dataconnect", `Not yet linked to the Cloud SQL instance.`); + return; + } + // TODO: make this prompt + // Should we upsert service here as well? so `database:sql:migrate` work for new service as well. + logLabeledBullet("dataconnect", `Linking the Cloud SQL instance...`); + // If no schema has been deployed yet, deploy an empty one to get connectivity. + currentSchema = { + name: `${serviceName}/schemas/${SCHEMA_ID}`, + source: { + files: [], + }, + primaryDatasource: { + postgresql: { + database: databaseId, + cloudSql: { + instance: instanceId, + }, + }, + }, + }; } - if ( - !currentSchema.primaryDatasource.postgresql || - currentSchema.primaryDatasource.postgresql.schemaValidation === "STRICT" - ) { + const postgresql = currentSchema.primaryDatasource.postgresql; + if (postgresql?.cloudSql.instance !== instanceId) { + logLabeledWarning( + "dataconnect", + `Switching connected Cloud SQL instance\nFrom ${postgresql?.cloudSql.instance}\nTo ${instanceId}`, + ); + } + if (postgresql?.database !== databaseId) { + logLabeledWarning( + "dataconnect", + `Switching connected Postgres database from ${postgresql?.database} to ${databaseId}`, + ); + } + if (!postgresql || postgresql.schemaValidation === "STRICT") { return; } - currentSchema.primaryDatasource.postgresql.schemaValidation = "STRICT"; + postgresql.schemaValidation = "STRICT"; try { await upsertSchema(currentSchema, /** validateOnly=*/ false); } catch (err: any) { @@ -315,11 +356,31 @@ async function ensureServiceIsConnectedToCloudSql( } function displaySchemaChanges(error: IncompatibleSqlSchemaError) { - const message = - "Your new schema is incompatible with the schema of your CloudSQL database. " + - "The following SQL statements will migrate your database schema to match your new Data Connect schema.\n" + - error.diffs.map(toString).join("\n"); - logLabeledWarning("dataconnect", message); + switch (error.violationType) { + case "INCOMPATIBLE_SCHEMA": + { + const message = + "Your new schema is incompatible with the schema of your CloudSQL database. " + + "The following SQL statements will migrate your database schema to match your new Data Connect schema.\n" + + error.diffs.map(toString).join("\n"); + logLabeledWarning("dataconnect", message); + } + break; + case "INACCESSIBLE_SCHEMA": + { + const message = + "Cannot access your CloudSQL database to validate schema. " + + "The following SQL statements can setup a new database schema.\n" + + error.diffs.map(toString).join("\n"); + logLabeledWarning("dataconnect", message); + logLabeledWarning("dataconnect", "Some SQL resources may already exist."); + } + break; + default: + throw new FirebaseError( + `Unknown schema violation type: ${error.violationType}, IncompatibleSqlSchemaError: ${error}`, + ); + } } function toString(diff: Diff) { diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index 7efaee5405f..c2d776fcc58 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -48,12 +48,15 @@ export interface File { content: string; } -// An error indicating that the SQL database schema is incomptible with a data connect schema. +// An error indicating that the SQL database schema is incompatible with a data connect schema. export interface IncompatibleSqlSchemaError { - // A list of differences between the two schema with instrucitons how to resolve them. + // A list of differences between the two schema with instructions how to resolve them. diffs: Diff[]; // Whether any of the changes included are destructive. destructive: boolean; + + // The failed precondition validation type. + violationType: "INCOMPATIBLE_SCHEMA" | "INACCESSIBLE_SCHEMA" | string; } export interface Diff { diff --git a/src/gcp/cloudsql/cloudsqladmin.ts b/src/gcp/cloudsql/cloudsqladmin.ts index c61c0aaeb86..b5580aa7b2f 100755 --- a/src/gcp/cloudsql/cloudsqladmin.ts +++ b/src/gcp/cloudsql/cloudsqladmin.ts @@ -51,6 +51,8 @@ export async function createInstance( userLabels: { "firebase-data-connect": "ft" }, insightsConfig: { queryInsightsEnabled: true, + queryPlansPerMinute: 5, // Match the default settings + queryStringLength: 1024, // Match the default settings }, }, }); diff --git a/src/gcp/cloudsql/connect.ts b/src/gcp/cloudsql/connect.ts index 5068b8b7d01..5bb78a1b7bc 100644 --- a/src/gcp/cloudsql/connect.ts +++ b/src/gcp/cloudsql/connect.ts @@ -28,11 +28,11 @@ export async function execute( const connectionName = instance.connectionName; if (!connectionName) { throw new FirebaseError( - `Could not get instance conection string for ${opts.instanceId}:${opts.databaseId}`, + `Could not get instance connection string for ${opts.instanceId}:${opts.databaseId}`, ); } let connector: Connector; - let pool: pg.Pool; + let client: pg.Client; switch (user.type) { case "CLOUD_IAM_USER": { connector = new Connector({ @@ -43,11 +43,10 @@ export async function execute( ipType: IpAddressTypes.PUBLIC, authType: AuthTypes.IAM, }); - pool = new pg.Pool({ + client = new pg.Client({ ...clientOpts, user: opts.username, database: opts.databaseId, - max: 1, }); break; } @@ -61,16 +60,15 @@ export async function execute( ipType: IpAddressTypes.PUBLIC, authType: AuthTypes.IAM, }); - pool = new pg.Pool({ + client = new pg.Client({ ...clientOpts, user: opts.username, database: opts.databaseId, - max: 1, }); break; } default: { - // cSQL doesn't return user.type for BUILT_IN users... + // Cloud SQL doesn't return user.type for BUILT_IN users... if (!opts.password) { throw new FirebaseError(`Cannot connect as BUILT_IN user without a password.`); } @@ -81,27 +79,27 @@ export async function execute( instanceConnectionName: connectionName, ipType: IpAddressTypes.PUBLIC, }); - pool = new pg.Pool({ + client = new pg.Client({ ...clientOpts, user: opts.username, password: opts.password, database: opts.databaseId, - max: 1, }); break; } } + logFn(`Logged in as ${opts.username}`); for (const s of sqlStatements) { - logFn(`Executing: '${s}' as ${opts.username}`); + logFn(`Executing: '${s}'`); try { - await pool.query(s); + await client.query(s); } catch (err) { throw new FirebaseError(`Error executing ${err}`); } } - await pool.end(); + await client.end(); connector.close(); } diff --git a/src/gcp/cloudsql/types.ts b/src/gcp/cloudsql/types.ts index e4fb71e0879..76cee9748cc 100644 --- a/src/gcp/cloudsql/types.ts +++ b/src/gcp/cloudsql/types.ts @@ -56,6 +56,8 @@ export interface DatabaseFlag { interface InsightsConfig { queryInsightsEnabled: boolean; + queryPlansPerMinute: number; + queryStringLength: number; } // TODO: Consider splitting off return only fields and input fields into different types. diff --git a/src/requireAuth.ts b/src/requireAuth.ts index 293d6924821..09329fc058d 100644 --- a/src/requireAuth.ts +++ b/src/requireAuth.ts @@ -37,6 +37,9 @@ function getAuthClient(config: GoogleAuthOptions): GoogleAuth { * @param authScopes scopes to be obtained. */ async function autoAuth(options: Options, authScopes: string[]): Promise { + if (process.env.MONOSPACE_ENV) { + throw new FirebaseError("autoAuth not yet implemented for IDX. Please run 'firebase login'"); + } const client = getAuthClient({ scopes: authScopes, projectId: options.project }); const token = await client.getAccessToken(); token !== null ? apiv2.setAccessToken(token) : false;