NEOS-409, NEOS-407: retain transformer data across pages

frontend/app/jobs/[id]/source/components/DataGenConnectionCard.tsx

@@ -57,20 +57,12 @@ interface Props {
 
 export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
   const { toast } = useToast();
-  // const { account } = useAccount();
+
   const { data, mutate, isLoading: isJobLoading } = useGetJob(jobId);
   const fkSourceConnectionId = getFkIdFromGenerateSource(data?.job?.source);
   const { data: schema, isLoading: isGetConnectionsSchemaLoading } =
     useGetConnectionSchema(fkSourceConnectionId);
 
-  // const { data: st } = useGetSystemTransformers();
-  // const { data: udt } = useGetUserDefinedTransformers(account?.id ?? '');
-
-  // const udts = udt?.transformers ?? [];
-  // const sts = st?.transformers ?? [];
-
-  // const merged = MergeSystemAndCustomTransformers(sts, udts);
-
   const form = useForm<SingleTableSchemaFormValues>({
     resolver: yupResolver(SINGLE_TABLE_SCHEMA_FORM_SCHEMA),
     defaultValues: {
@@ -114,7 +106,9 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
     }
   }
 
-  const formValues = form.watch();
+  // changed to getValues() from watch because watch would listen for a change and then trigger a render which would overwrite the values that were changed (which we wanted to keep)
+
+  const formValues = form.getValues();
   const schemaTableData = formValues.mappings?.map((mapping) => ({
     ...mapping,
     schema: formValues.schema,
@@ -226,7 +220,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
         {formValues.schema && formValues.table && (
           <SchemaTable data={schemaTableData} excludeTransformers />
         )}
-        <div className="flex flex-row gap-1 justify-between">
+        <div className="flex flex-row gap-1 justify-end">
           <Button key="submit" type="submit">
             Submit
           </Button>
@@ -321,7 +315,6 @@ function getJobSource(
 
   const mappings: SingleTableSchemaFormValues['mappings'] = dbCols.map((c) => {
     const colMapping = getColumnMapping(schemaMap, c.schema, c.table, c.column);
-
     const transformer =
       colMapping?.transformer ??
       new JobMappingTransformer({

frontend/app/jobs/[id]/source/components/DataSyncConnectionCard.tsx

@@ -1,10 +1,7 @@
 'use client';
 import SourceOptionsForm from '@/components/jobs/Form/SourceOptionsForm';
-import { FindTransformerByName } from '@/components/jobs/SchemaTable/TransformerSelect';
 import {
-  MergeSystemAndCustomTransformers,
   SchemaTable,
-  TransformerWithType,
   getConnectionSchema,
 } from '@/components/jobs/SchemaTable/schema-table';
 import { useAccount } from '@/components/providers/account-provider';
@@ -31,8 +28,6 @@ import { useToast } from '@/components/ui/use-toast';
 import { useGetConnectionSchema } from '@/libs/hooks/useGetConnectionSchema';
 import { useGetConnections } from '@/libs/hooks/useGetConnections';
 import { useGetJob } from '@/libs/hooks/useGetJob';
-import { useGetSystemTransformers } from '@/libs/hooks/useGetSystemTransformers';
-import { useGetUserDefinedTransformers } from '@/libs/hooks/useGetUserDefinedTransformers';
 import {
   Connection,
   DatabaseColumn,
@@ -104,14 +99,6 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
 
   const connections = connectionsData?.connections ?? [];
 
-  const { data: st } = useGetSystemTransformers();
-  const { data: udt } = useGetUserDefinedTransformers(account?.id ?? '');
-
-  const udts = udt?.transformers ?? [];
-  const sts = st?.transformers ?? [];
-
-  const merged = MergeSystemAndCustomTransformers(sts, udts);
-
   const form = useForm({
     resolver: yupResolver<SourceFormValues>(FORM_SCHEMA),
     defaultValues: {
@@ -122,12 +109,12 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
       destinationIds: [],
       mappings: [],
     },
-    values: getJobSource(merged, data?.job, schema?.schemas),
+    values: getJobSource(data?.job, schema?.schemas),
   });
 
   async function onSourceChange(value: string): Promise<void> {
     try {
-      const newValues = await getUpdatedValues(value, form.getValues(), merged);
+      const newValues = await getUpdatedValues(value, form.getValues());
       form.reset(newValues);
     } catch (err) {
       form.reset({ ...form.getValues, mappings: [], sourceId: value });
@@ -334,11 +321,7 @@ function getExistingMysqlSourceConnectionOptions(
     : undefined;
 }
 
-function getJobSource(
-  merged: TransformerWithType[],
-  job?: Job,
-  schema?: DatabaseColumn[]
-): SourceFormValues {
+function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
   if (!job || !schema) {
     return {
       sourceId: '',
@@ -404,6 +387,7 @@ function getJobSource(
       };
     }
   });
+
   const mappings = schema.map((c) => {
     const colMapping = getColumnMapping(schemaMap, c.schema, c.table, c.column);
 
@@ -438,10 +422,7 @@ function getJobSource(
     sourceId: getConnectionIdFromSource(job.source) || '',
     mappings: values.mappings.map((mapping) => ({
       ...mapping,
-      transformer: FindTransformerByName(
-        mapping?.transformer.name ?? '',
-        merged
-      ) as {
+      transformer: mapping.transformer as {
         name: string;
         source: string;
         config: { config: { case: string; value: {} } };
@@ -491,8 +472,7 @@ export function getColumnMapping(
 
 async function getUpdatedValues(
   connectionId: string,
-  originalValues: SourceFormValues,
-  merged: TransformerWithType[]
+  originalValues: SourceFormValues
 ): Promise<SourceFormValues> {
   const [schemaRes, connRes] = await Promise.all([
     getConnectionSchema(connectionId),
@@ -521,10 +501,7 @@ async function getUpdatedValues(
     ...values,
     mappings: values.mappings.map((mapping) => ({
       ...mapping,
-      transformer: FindTransformerByName(
-        mapping?.transformer.name ?? 'passthrough',
-        merged
-      ) as {
+      transformer: mapping.transformer as {
         name: string;
         source: string;
         config: { config: { case: string; value: {} } };

frontend/app/new/job/connect/page.tsx

@@ -66,6 +66,7 @@ export default function Page({ searchParams }: PageProps): ReactElement {
     control: form.control,
     name: 'destinations',
   });
+
   useFormPersist(`${sessionPrefix}-new-job-connect`, {
     watch: form.watch,
     setValue: form.setValue,

frontend/app/new/job/schema/page.tsx

@@ -49,9 +49,12 @@ export default function Page({ searchParams }: PageProps): ReactElement {
     }
   );
 
-  useSessionStorage<SchemaFormValues>(`${sessionPrefix}-new-job-schema`, {
-    mappings: [],
-  });
+  const [schemaFormData] = useSessionStorage<SchemaFormValues>(
+    `${sessionPrefix}-new-job-schema`,
+    {
+      mappings: [],
+    }
+  );
 
   async function getSchema(): Promise<SchemaFormValues> {
     try {
@@ -60,33 +63,57 @@ export default function Page({ searchParams }: PageProps): ReactElement {
         return { mappings: [] };
       }
 
-      const mappings = res.schemas.map((r) => {
-        var pt = new JobMappingTransformer({
-          source: 'passthrough',
-          name: 'passthrough',
-          config: new TransformerConfig({
+      // set values from the session data if they're available
+      // this helps retain data from page to page and across saves before the data is submitted
+      if (schemaFormData.mappings.length > 0) {
+        const mappings = schemaFormData.mappings.map((r) => {
+          var pt = JobMappingTransformer.fromJson(r.transformer) as {
+            source: string;
+            name: string;
             config: {
-              case: 'passthroughConfig',
-              value: new Passthrough({}),
-            },
-          }),
-        }) as {
-          source: string;
-          name: string;
-          config: {
+              config: {
+                case?: string;
+                value: {};
+              };
+            };
+          };
+
+          return {
+            ...r,
+            transformer: pt,
+          };
+        });
+
+        return { mappings };
+      } else {
+        const mappings = res.schemas.map((r) => {
+          var pt = new JobMappingTransformer({
+            source: 'passthrough',
+            name: 'passthrough',
+            config: new TransformerConfig({
+              config: {
+                case: 'passthroughConfig',
+                value: new Passthrough({}),
+              },
+            }),
+          }) as {
+            source: string;
+            name: string;
             config: {
-              case?: string;
-              value: {};
+              config: {
+                case?: string;
+                value: {};
+              };
             };
           };
-        };
 
-        return {
-          ...r,
-          transformer: pt,
-        };
-      });
-      return { mappings };
+          return {
+            ...r,
+            transformer: pt,
+          };
+        });
+        return { mappings };
+      }
     } catch (err) {
       console.error(err);
       toast({

frontend/app/transformers/Sheetforms/GenerateFloatForm.tsx

@@ -117,8 +117,10 @@ export default function GenerateFloatForm(props: Props): ReactElement {
               <Input
                 className="max-w-[180px]"
                 placeholder="3"
+                type="number"
                 max={9}
-                value={bd}
+                min={1}
+                value={String(bd)}
                 onChange={(event) => {
                   const inputValue = Math.min(
                     9,
@@ -148,8 +150,10 @@ export default function GenerateFloatForm(props: Props): ReactElement {
               <Input
                 className="max-w-[180px]"
                 placeholder="3"
+                type="number"
                 max={9}
-                value={ad}
+                min={1}
+                value={String(ad)}
                 onChange={(event) => {
                   const inputValue = Math.min(
                     9,

frontend/components/jobs/SchemaTable/TransformerSelect.tsx

@@ -131,9 +131,7 @@ export default function TransformerSelect(props: Props): ReactElement {
                               FindTransformerByName(currentValue, transformers)
                                 ?.source ?? '',
                             name: selectedTransformer?.name,
-                            config: new TransformerConfig({
-                              config: selectedTransformer?.config?.config,
-                            }),
+                            config: selectedTransformer?.config,
                           }
                         );
                         onSelect(jobMappingTransformer);
@@ -165,7 +163,7 @@ export default function TransformerSelect(props: Props): ReactElement {
   );
 }
 
-export function FindTransformerByName(
+function FindTransformerByName(
   name: string,
   transformers: TransformerWithType[]
 ): TransformerWithType | undefined {

frontend/components/jobs/SchemaTable/schema-table.tsx

@@ -84,7 +84,7 @@ export async function getConnectionSchema(
 export type TransformerWithType = SystemTransformer &
   UserDefinedTransformer & { transformerType: 'system' | 'custom' };
 
-export function MergeSystemAndCustomTransformers(
+function MergeSystemAndCustomTransformers(
   system: SystemTransformer[],
   custom: UserDefinedTransformer[]
 ): TransformerWithType[] {