|
45 | 45 |
|
46 | 46 | #: For taps, support emitting BATCH messages. For targets, support consuming BATCH
|
47 | 47 | #: messages.
|
| 48 | +#: |
| 49 | +#: Example: |
| 50 | +#: |
| 51 | +#: .. code-block:: json |
| 52 | +#: |
| 53 | +#: { |
| 54 | +#: "batch_config": { |
| 55 | +#: "encoding": { |
| 56 | +#: "format": "jsonl", |
| 57 | +#: "compression": "gzip" |
| 58 | +#: }, |
| 59 | +#: "storage": { |
| 60 | +#: "type": "root", |
| 61 | +#: "root": "file:///path/to/batch/files", |
| 62 | +#: "prefix": "batch-" |
| 63 | +#: } |
| 64 | +#: } |
| 65 | +#: } |
| 66 | +#: |
48 | 67 | BATCH = Builtin(
|
49 | 68 | schema=schema.BATCH_CONFIG,
|
50 | 69 | capability=PluginCapabilities.BATCH,
|
51 | 70 | )
|
52 | 71 |
|
| 72 | +#: Support schema flattening, aka de-nesting of complex properties. |
| 73 | +#: |
| 74 | +#: Example: |
| 75 | +#: |
| 76 | +#: .. code-block:: json |
| 77 | +#: |
| 78 | +#: { |
| 79 | +#: "flattening_enabled": true, |
| 80 | +#: "flattening_max_depth": 3 |
| 81 | +#: } |
| 82 | +#: |
53 | 83 | FLATTENING = Builtin(
|
54 | 84 | schema=schema.FLATTENING_CONFIG,
|
55 | 85 | capability=PluginCapabilities.FLATTENING,
|
56 | 86 | )
|
| 87 | + |
| 88 | +#: Support inline stream map transforms. |
| 89 | +#: |
| 90 | +#: Example: |
| 91 | +#: |
| 92 | +#: .. code-block:: json |
| 93 | +#: |
| 94 | +#: { |
| 95 | +#: "stream_maps": { |
| 96 | +#: "users": { |
| 97 | +#: "id": "id", |
| 98 | +#: "fields": "[f for f in fields if f['key'] != 'age']" |
| 99 | +#: } |
| 100 | +#: } |
| 101 | +#: } |
| 102 | +#: |
57 | 103 | STREAM_MAPS = Builtin(
|
58 | 104 | schema.STREAM_MAPS_CONFIG,
|
59 | 105 | capability=PluginCapabilities.STREAM_MAPS,
|
60 | 106 | )
|
| 107 | + |
| 108 | +#: Target batch size in rows. |
| 109 | +#: |
| 110 | +#: Example: |
| 111 | +#: |
| 112 | +#: .. code-block:: json |
| 113 | +#: |
| 114 | +#: { |
| 115 | +#: "batch_size_rows": 10000 |
| 116 | +#: } |
| 117 | +#: |
61 | 118 | TARGET_BATCH_SIZE_ROWS = Builtin(schema=schema.TARGET_BATCH_SIZE_ROWS_CONFIG)
|
| 119 | + |
| 120 | +#: Support hard delete capability. |
| 121 | +#: |
| 122 | +#: Example: |
| 123 | +#: |
| 124 | +#: .. code-block:: json |
| 125 | +#: |
| 126 | +#: { |
| 127 | +#: "hard_delete": true |
| 128 | +#: } |
| 129 | +#: |
62 | 130 | TARGET_HARD_DELETE = Builtin(
|
63 | 131 | schema=schema.TARGET_HARD_DELETE_CONFIG,
|
64 | 132 | capability=TargetCapabilities.HARD_DELETE,
|
65 | 133 | )
|
| 134 | + |
| 135 | +#: Target load method. |
| 136 | +#: |
| 137 | +#: Example: |
| 138 | +#: |
| 139 | +#: .. code-block:: json |
| 140 | +#: |
| 141 | +#: { |
| 142 | +#: "load_method": "upsert" |
| 143 | +#: } |
| 144 | +#: |
66 | 145 | TARGET_LOAD_METHOD = Builtin(schema=schema.TARGET_LOAD_METHOD_CONFIG)
|
| 146 | + |
| 147 | +#: Allow setting the target schema. |
| 148 | +#: |
| 149 | +#: Example: |
| 150 | +#: |
| 151 | +#: .. code-block:: json |
| 152 | +#: |
| 153 | +#: { |
| 154 | +#: "default_target_schema": "my_schema" |
| 155 | +#: } |
| 156 | +#: |
67 | 157 | TARGET_SCHEMA = Builtin(
|
68 | 158 | schema=schema.TARGET_SCHEMA_CONFIG,
|
69 | 159 | capability=TargetCapabilities.TARGET_SCHEMA,
|
70 | 160 | )
|
| 161 | + |
| 162 | +#: Validate incoming records against their declared schema. |
| 163 | +#: |
| 164 | +#: Example: |
| 165 | +#: |
| 166 | +#: .. code-block:: json |
| 167 | +#: |
| 168 | +#: { |
| 169 | +#: "validate_records": true |
| 170 | +#: } |
| 171 | +#: |
71 | 172 | TARGET_VALIDATE_RECORDS = Builtin(
|
72 | 173 | schema=schema.TARGET_VALIDATE_RECORDS_CONFIG,
|
73 | 174 | capability=TargetCapabilities.VALIDATE_RECORDS,
|
|
0 commit comments