feature: add tag_regexes flag into the frontend

Author: cdxker

clients/ts-sdk/openapi.json

@@ -8170,6 +8170,13 @@
             "type": "boolean",
             "description": "This option will ingest all variants as individual chunks and place them in groups by product id. Turning this off will only scrape 1 variant per product. default: true",
             "nullable": true
+          },
+          "tag_regexes": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "nullable": true
           }
         }
       },

clients/ts-sdk/src/types.gen.ts

@@ -610,6 +610,7 @@ export type CrawlShopifyOptions = {
      * This option will ingest all variants as individual chunks and place them in groups by product id. Turning this off will only scrape 1 variant per product. default: true
      */
     group_variants?: (boolean) | null;
+    tag_regexes?: Array<(string)> | null;
 };
 
 export type CreateApiKeyReqPayload = {

frontends/dashboard/src/pages/dataset/CrawlingSettings.tsx

@@ -37,6 +37,7 @@ export type FlatCrawlOptions = Omit<CrawlOptions, "scrape_options"> & {
   openapi_schema_url?: string;
   openapi_tag?: string;
   group_variants?: boolean | null;
+  tag_regexes?: string[] | null;
 };
 
 export const unflattenCrawlOptions = (
@@ -79,6 +80,7 @@ export const unflattenCrawlOptions = (
       scrape_options: {
         type: "shopify",
         group_variants: options.group_variants,
+        tag_regexes: options.tag_regexes ?? [],
       },
     };
   }
@@ -111,6 +113,7 @@ export const flattenCrawlOptions = (
       ...options,
       type: "shopify",
       group_variants: options.scrape_options.group_variants,
+      tag_regexes: options.scrape_options.tag_regexes,
     };
   } else {
     return {
@@ -426,7 +429,7 @@ const RealCrawlingSettings = (props: RealCrawlingSettingsProps) => {
 
       <div class="flex items-center gap-2 py-2 pt-4">
         <Show when={isShopify()}>
-          <label class="block pl-4">Group Product Variants?</label>
+          <label class="block">Group Product Variants?</label>
           <input
             onChange={(e) =>
               setOptions("group_variants", e.currentTarget.checked)
@@ -437,6 +440,28 @@ const RealCrawlingSettings = (props: RealCrawlingSettingsProps) => {
           />
         </Show>
       </div>
+      <div class="items-center gap-2 py-2 pt-4">
+        <Show when={isShopify()}>
+          <div class="flex items-center gap-2">
+            <div>Important Product Tags (regex)</div>
+            <Tooltip
+              tooltipText="Regex pattern of tags to use from the Shopify API, e.g. 'Men' to include 'Men' if it exists in a product tag."
+              body={<FaRegularCircleQuestion class="h-3 w-3 text-black" />}
+            />
+          </div>
+          <MultiStringInput
+            disabled={!isShopify()}
+            placeholder="Men"
+            addClass="bg-magenta-100/40 px-2 text-sm rounded border border-magenta-300/40"
+            addLabel="Add Product Tag"
+            onChange={(value) => {
+              setOptions("tag_regexes", value);
+            }}
+            value={options.tag_regexes || []}
+          />
+          <Error error={errors.tag_regexes} />
+        </Show>
+      </div>
 
       <div classList={{ "flex gap-4 pt-2": true, "opacity-40": isShopify() }}>
         <div>