From 5eec6b671fbfcf6155b6fb207c3c232166a05742 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 00:53:23 -0800 Subject: [PATCH 1/5] retain transformer data across pages --- .../components/DataGenConnectionCard.tsx | 12 +-- frontend/app/new/job/connect/page.tsx | 1 + frontend/app/new/job/schema/page.tsx | 79 +++++++++++++------ .../Sheetforms/GenerateFloatForm.tsx | 10 ++- .../jobs/SchemaTable/TransformerSelect.tsx | 4 +- 5 files changed, 67 insertions(+), 39 deletions(-) diff --git a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx index 9881b0e177..b0ae331083 100644 --- a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -57,20 +57,12 @@ interface Props { export default function DataGenConnectionCard({ jobId }: Props): ReactElement { const { toast } = useToast(); - // const { account } = useAccount(); + const { data, mutate, isLoading: isJobLoading } = useGetJob(jobId); const fkSourceConnectionId = getFkIdFromGenerateSource(data?.job?.source); const { data: schema, isLoading: isGetConnectionsSchemaLoading } = useGetConnectionSchema(fkSourceConnectionId); - // const { data: st } = useGetSystemTransformers(); - // const { data: udt } = useGetUserDefinedTransformers(account?.id ?? ''); - - // const udts = udt?.transformers ?? []; - // const sts = st?.transformers ?? []; - - // const merged = MergeSystemAndCustomTransformers(sts, udts); - const form = useForm({ resolver: yupResolver(SINGLE_TABLE_SCHEMA_FORM_SCHEMA), defaultValues: { @@ -128,6 +120,8 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { const selectedSchemaTables = schemaTableMap.get(formValues.schema) ?? []; + console.log('form', form.getValues()); + return (
diff --git a/frontend/app/new/job/connect/page.tsx b/frontend/app/new/job/connect/page.tsx index 265e241d01..e1ec55b0b4 100644 --- a/frontend/app/new/job/connect/page.tsx +++ b/frontend/app/new/job/connect/page.tsx @@ -66,6 +66,7 @@ export default function Page({ searchParams }: PageProps): ReactElement { control: form.control, name: 'destinations', }); + useFormPersist(`${sessionPrefix}-new-job-connect`, { watch: form.watch, setValue: form.setValue, diff --git a/frontend/app/new/job/schema/page.tsx b/frontend/app/new/job/schema/page.tsx index e542d8957a..4c456beceb 100644 --- a/frontend/app/new/job/schema/page.tsx +++ b/frontend/app/new/job/schema/page.tsx @@ -49,9 +49,12 @@ export default function Page({ searchParams }: PageProps): ReactElement { } ); - useSessionStorage(`${sessionPrefix}-new-job-schema`, { - mappings: [], - }); + const [schemaFormData] = useSessionStorage( + `${sessionPrefix}-new-job-schema`, + { + mappings: [], + } + ); async function getSchema(): Promise { try { @@ -60,33 +63,57 @@ export default function Page({ searchParams }: PageProps): ReactElement { return { mappings: [] }; } - const mappings = res.schemas.map((r) => { - var pt = new JobMappingTransformer({ - source: 'passthrough', - name: 'passthrough', - config: new TransformerConfig({ + // set values from the session data if they're available + // this helps retain data from page to page and across saves before the data is submitted + if (schemaFormData.mappings.length > 0) { + const mappings = schemaFormData.mappings.map((r) => { + var pt = JobMappingTransformer.fromJson(r.transformer) as { + source: string; + name: string; config: { - case: 'passthroughConfig', - value: new Passthrough({}), - }, - }), - }) as { - source: string; - name: string; - config: { + config: { + case?: string; + value: {}; + }; + }; + }; + + return { + ...r, + transformer: pt, + }; + }); + + return { mappings }; + } else { + const mappings = res.schemas.map((r) => { + var pt = new JobMappingTransformer({ + source: 'passthrough', + name: 'passthrough', + config: new TransformerConfig({ + config: { + case: 'passthroughConfig', + value: new Passthrough({}), + }, + }), + }) as { + source: string; + name: string; config: { - case?: string; - value: {}; + config: { + case?: string; + value: {}; + }; }; }; - }; - return { - ...r, - transformer: pt, - }; - }); - return { mappings }; + return { + ...r, + transformer: pt, + }; + }); + return { mappings }; + } } catch (err) { console.error(err); toast({ @@ -120,6 +147,8 @@ export default function Page({ searchParams }: PageProps): ReactElement { router.push(`/new/job/subset?sessionId=${sessionPrefix}`); } + console.log('form data', form.getValues()); + return (
{ const inputValue = Math.min( 9, @@ -148,8 +152,10 @@ export default function GenerateFloatForm(props: Props): ReactElement { { const inputValue = Math.min( 9, diff --git a/frontend/components/jobs/SchemaTable/TransformerSelect.tsx b/frontend/components/jobs/SchemaTable/TransformerSelect.tsx index 2048eed37f..9cf5492aff 100644 --- a/frontend/components/jobs/SchemaTable/TransformerSelect.tsx +++ b/frontend/components/jobs/SchemaTable/TransformerSelect.tsx @@ -131,9 +131,7 @@ export default function TransformerSelect(props: Props): ReactElement { FindTransformerByName(currentValue, transformers) ?.source ?? '', name: selectedTransformer?.name, - config: new TransformerConfig({ - config: selectedTransformer?.config?.config, - }), + config: selectedTransformer?.config, } ); onSelect(jobMappingTransformer); From 222f8d2bb241d716f51817a5ff559c97c713fd49 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 11:16:09 -0800 Subject: [PATCH 2/5] working data sync source transformers --- .../jobs/[id]/source/components/DataGenConnectionCard.tsx | 6 +++++- .../[id]/source/components/DataSyncConnectionCard.tsx | 8 ++++---- .../app/transformers/Sheetforms/GenerateFloatForm.tsx | 2 -- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx index b0ae331083..b383d613e8 100644 --- a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -258,6 +258,7 @@ function getJobSource( } const schemaMap: SchemaMap = {}; + console.log('job', job.mappings); job?.mappings.forEach((c) => { if (!schemaMap[c.schema]) { schemaMap[c.schema] = { @@ -315,7 +316,7 @@ function getJobSource( const mappings: SingleTableSchemaFormValues['mappings'] = dbCols.map((c) => { const colMapping = getColumnMapping(schemaMap, c.schema, c.table, c.column); - + console.log('c', colMapping); const transformer = colMapping?.transformer ?? new JobMappingTransformer({ @@ -340,6 +341,8 @@ function getJobSource( }; }); + console.log('mappings', mappings); + return { mappings: mappings, numRows: numRows, @@ -361,6 +364,7 @@ async function updateJobConnection( new UpdateJobSourceConnectionRequest({ id: job.id, mappings: values.mappings.map((m) => { + console.log('m', m.transformer); const jmt = new JobMappingTransformer({ source: m.transformer.source, name: m.transformer.name, diff --git a/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx index f392218867..972dc14acc 100644 --- a/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx @@ -174,6 +174,8 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { const source = connections.find((item) => item.id == sourceConnectionId); + console.log('mappings', form.getValues()); + return ( @@ -404,6 +406,7 @@ function getJobSource( }; } }); + const mappings = schema.map((c) => { const colMapping = getColumnMapping(schemaMap, c.schema, c.table, c.column); @@ -438,10 +441,7 @@ function getJobSource( sourceId: getConnectionIdFromSource(job.source) || '', mappings: values.mappings.map((mapping) => ({ ...mapping, - transformer: FindTransformerByName( - mapping?.transformer.name ?? '', - merged - ) as { + transformer: mapping.transformer as { name: string; source: string; config: { config: { case: string; value: {} } }; diff --git a/frontend/app/transformers/Sheetforms/GenerateFloatForm.tsx b/frontend/app/transformers/Sheetforms/GenerateFloatForm.tsx index c6119521f9..1c9c683d5a 100644 --- a/frontend/app/transformers/Sheetforms/GenerateFloatForm.tsx +++ b/frontend/app/transformers/Sheetforms/GenerateFloatForm.tsx @@ -63,8 +63,6 @@ export default function GenerateFloatForm(props: Props): ReactElement { const signs = ['positive', 'negative', 'random']; - console.log('bd', bdValue, adValue, sign); - return (
Date: Mon, 4 Dec 2023 12:05:04 -0800 Subject: [PATCH 3/5] working data gen source connection form --- .../components/DataGenConnectionCard.tsx | 9 +---- .../components/DataSyncConnectionCard.tsx | 33 +++---------------- frontend/app/new/job/schema/page.tsx | 2 -- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx index b383d613e8..9b60d24d2b 100644 --- a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -106,7 +106,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { } } - const formValues = form.watch(); + const formValues = form.getValues(); const schemaTableData = formValues.mappings?.map((mapping) => ({ ...mapping, schema: formValues.schema, @@ -120,8 +120,6 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { const selectedSchemaTables = schemaTableMap.get(formValues.schema) ?? []; - console.log('form', form.getValues()); - return ( @@ -258,7 +256,6 @@ function getJobSource( } const schemaMap: SchemaMap = {}; - console.log('job', job.mappings); job?.mappings.forEach((c) => { if (!schemaMap[c.schema]) { schemaMap[c.schema] = { @@ -316,7 +313,6 @@ function getJobSource( const mappings: SingleTableSchemaFormValues['mappings'] = dbCols.map((c) => { const colMapping = getColumnMapping(schemaMap, c.schema, c.table, c.column); - console.log('c', colMapping); const transformer = colMapping?.transformer ?? new JobMappingTransformer({ @@ -341,8 +337,6 @@ function getJobSource( }; }); - console.log('mappings', mappings); - return { mappings: mappings, numRows: numRows, @@ -364,7 +358,6 @@ async function updateJobConnection( new UpdateJobSourceConnectionRequest({ id: job.id, mappings: values.mappings.map((m) => { - console.log('m', m.transformer); const jmt = new JobMappingTransformer({ source: m.transformer.source, name: m.transformer.name, diff --git a/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx index 972dc14acc..e154fdd03b 100644 --- a/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx @@ -1,10 +1,7 @@ 'use client'; import SourceOptionsForm from '@/components/jobs/Form/SourceOptionsForm'; -import { FindTransformerByName } from '@/components/jobs/SchemaTable/TransformerSelect'; import { - MergeSystemAndCustomTransformers, SchemaTable, - TransformerWithType, getConnectionSchema, } from '@/components/jobs/SchemaTable/schema-table'; import { useAccount } from '@/components/providers/account-provider'; @@ -31,8 +28,6 @@ import { useToast } from '@/components/ui/use-toast'; import { useGetConnectionSchema } from '@/libs/hooks/useGetConnectionSchema'; import { useGetConnections } from '@/libs/hooks/useGetConnections'; import { useGetJob } from '@/libs/hooks/useGetJob'; -import { useGetSystemTransformers } from '@/libs/hooks/useGetSystemTransformers'; -import { useGetUserDefinedTransformers } from '@/libs/hooks/useGetUserDefinedTransformers'; import { Connection, DatabaseColumn, @@ -104,14 +99,6 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { const connections = connectionsData?.connections ?? []; - const { data: st } = useGetSystemTransformers(); - const { data: udt } = useGetUserDefinedTransformers(account?.id ?? ''); - - const udts = udt?.transformers ?? []; - const sts = st?.transformers ?? []; - - const merged = MergeSystemAndCustomTransformers(sts, udts); - const form = useForm({ resolver: yupResolver(FORM_SCHEMA), defaultValues: { @@ -122,12 +109,12 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { destinationIds: [], mappings: [], }, - values: getJobSource(merged, data?.job, schema?.schemas), + values: getJobSource(data?.job, schema?.schemas), }); async function onSourceChange(value: string): Promise { try { - const newValues = await getUpdatedValues(value, form.getValues(), merged); + const newValues = await getUpdatedValues(value, form.getValues()); form.reset(newValues); } catch (err) { form.reset({ ...form.getValues, mappings: [], sourceId: value }); @@ -174,8 +161,6 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { const source = connections.find((item) => item.id == sourceConnectionId); - console.log('mappings', form.getValues()); - return ( @@ -336,11 +321,7 @@ function getExistingMysqlSourceConnectionOptions( : undefined; } -function getJobSource( - merged: TransformerWithType[], - job?: Job, - schema?: DatabaseColumn[] -): SourceFormValues { +function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues { if (!job || !schema) { return { sourceId: '', @@ -491,8 +472,7 @@ export function getColumnMapping( async function getUpdatedValues( connectionId: string, - originalValues: SourceFormValues, - merged: TransformerWithType[] + originalValues: SourceFormValues ): Promise { const [schemaRes, connRes] = await Promise.all([ getConnectionSchema(connectionId), @@ -521,10 +501,7 @@ async function getUpdatedValues( ...values, mappings: values.mappings.map((mapping) => ({ ...mapping, - transformer: FindTransformerByName( - mapping?.transformer.name ?? 'passthrough', - merged - ) as { + transformer: mapping.transformer as { name: string; source: string; config: { config: { case: string; value: {} } }; diff --git a/frontend/app/new/job/schema/page.tsx b/frontend/app/new/job/schema/page.tsx index 4c456beceb..a117c9566f 100644 --- a/frontend/app/new/job/schema/page.tsx +++ b/frontend/app/new/job/schema/page.tsx @@ -147,8 +147,6 @@ export default function Page({ searchParams }: PageProps): ReactElement { router.push(`/new/job/subset?sessionId=${sessionPrefix}`); } - console.log('form data', form.getValues()); - return (
Date: Mon, 4 Dec 2023 12:07:38 -0800 Subject: [PATCH 4/5] clean up --- .../app/jobs/[id]/source/components/DataGenConnectionCard.tsx | 2 +- frontend/components/jobs/SchemaTable/TransformerSelect.tsx | 2 +- frontend/components/jobs/SchemaTable/schema-table.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx index 9b60d24d2b..a9414e2e55 100644 --- a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -218,7 +218,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { {formValues.schema && formValues.table && ( )} -
+
diff --git a/frontend/components/jobs/SchemaTable/TransformerSelect.tsx b/frontend/components/jobs/SchemaTable/TransformerSelect.tsx index 9cf5492aff..15cae807cc 100644 --- a/frontend/components/jobs/SchemaTable/TransformerSelect.tsx +++ b/frontend/components/jobs/SchemaTable/TransformerSelect.tsx @@ -163,7 +163,7 @@ export default function TransformerSelect(props: Props): ReactElement { ); } -export function FindTransformerByName( +function FindTransformerByName( name: string, transformers: TransformerWithType[] ): TransformerWithType | undefined { diff --git a/frontend/components/jobs/SchemaTable/schema-table.tsx b/frontend/components/jobs/SchemaTable/schema-table.tsx index 3d4ecdf891..86bacf9207 100644 --- a/frontend/components/jobs/SchemaTable/schema-table.tsx +++ b/frontend/components/jobs/SchemaTable/schema-table.tsx @@ -84,7 +84,7 @@ export async function getConnectionSchema( export type TransformerWithType = SystemTransformer & UserDefinedTransformer & { transformerType: 'system' | 'custom' }; -export function MergeSystemAndCustomTransformers( +function MergeSystemAndCustomTransformers( system: SystemTransformer[], custom: UserDefinedTransformer[] ): TransformerWithType[] { From 7d51b736ba3e747916bcbecb018eaa41cbbef871 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 12:08:59 -0800 Subject: [PATCH 5/5] comments --- .../app/jobs/[id]/source/components/DataGenConnectionCard.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx index a9414e2e55..e8bea6245d 100644 --- a/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -106,6 +106,8 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { } } + // changed to getValues() from watch because watch would listen for a change and then trigger a render which would overwrite the values that were changed (which we wanted to keep) + const formValues = form.getValues(); const schemaTableData = formValues.mappings?.map((mapping) => ({ ...mapping,