feat: Support filesystem interfaces

MeltanoLabs · Aug 24, 2024 · e39a403 · e39a403
1 parent 2675cec
commit e39a403
Show file tree

Hide file tree

Showing 5 changed files with 581 additions and 390 deletions.
diff --git a/meltano.yml b/meltano.yml
@@ -18,7 +18,62 @@ plugins:
         keys:
         - col1
       add_metadata_columns: false
+    settings_group_validation:
+      - [ftp.host]
+      - [github.org, github.repo]
     settings:
+      - name: filesystem
+        kind: options
+        options:
+          - label: Local Filesystem
+            value: local
+          - label: FTP
+            value: ftp
+          - label: GitHub
+            value: github
+
+      # FTP settings
+      - name: ftp.host
+        label: FTP Host
+        description: Hostname of the FTP server
+        kind: string
+      - name: ftp.port
+        label: FTP Port
+        description: Port of the FTP server
+        kind: integer
+      - name: ftp.username
+        label: FTP Username
+        description: Username for the FTP server
+        kind: string
+      - name: ftp.password
+        label: FTP Password
+        description: Password for the FTP server
+        kind: string
+        sensitive: true
+      - name: ftp.encoding
+        label: FTP Encoding
+        description: Encoding for the FTP server
+        kind: string
+
+      # GitHub settings
+      - name: github.org
+        label: GitHub Organization
+        description: Organization name on GitHub
+        kind: string
+      - name: github.repo
+        label: GitHub Repository
+        description: Repository name on GitHub
+        kind: string
+      - name: github.username
+        label: GitHub Username
+        description: Username for GitHub
+        kind: string
+      - name: github.token
+        label: GitHub Token
+        description: Token for GitHub
+        kind: string
+        sensitive: true
+
       - name: files
         description: Array of objects containing keys - `entity`, `path`, `keys`, `encoding` (Optional), `delimiter` (Optional), `doublequote` (Optional), `escapechar` (Optional), `quotechar` (Optional), `skipinitialspace` (Optional), `strict` (Optional)
         kind: array
@@ -30,6 +85,7 @@ plugins:
       - name: add_metadata_columns
         description: When True, add the metadata columns (`_sdc_source_file`, `_sdc_source_file_mtime`, `_sdc_source_lineno`) to output.
         kind: boolean
+
   loaders:
   - name: target-jsonl
     variant: andyh1203