diff --git a/docs/FTPSource-batchsource.md b/docs/FTPSource-batchsource.md index 56d59b0..a240adf 100644 --- a/docs/FTPSource-batchsource.md +++ b/docs/FTPSource-batchsource.md @@ -28,7 +28,7 @@ Properties **Password:** Password to use for authentication. **Format:** Format of the data to read. -The format must be one of 'blob', 'csv', 'delimited', 'json', 'text', 'tsv', or the +The format must be one of 'blob', 'csv', 'delimited', 'json', 'text', 'tsv', 'xls', or the name of any format plugin that you have deployed to your environment. Note that FTP does not support seeking in a file, so formats like avro and parquet cannot be used. If the format is a macro, only the formats listed above can be used. @@ -36,7 +36,20 @@ If the format is 'blob', every input file will be read into a separate record. The 'blob' format also requires a schema that contains a field named 'body' of type 'bytes'. If the format is 'text', the schema must contain a field named 'body' of type 'string'. -**Get Schema:** Auto-detects schema from file. Supported formats are: csv, delimited, tsv, blob and text. +**Sample Size:** The maximum number of rows that will get investigated for automatic data type detection. +The default value is 1000. + +**Override:** A list of columns with the corresponding data types for whom the automatic data type detection gets +skipped. + +**Terminate If Empty Row:** Specify whether to stop reading after encountering the first empty row. Defaults to false. + +**Select Sheet Using:** Select the sheet by name or number. Default is 'Sheet Number'. + +**Sheet Value:** The name/number of the sheet to read from. If not specified, the first sheet will be read. +Sheet Number are 0 based, ie first sheet is 0. + +**Get Schema:** Auto-detects schema from file. Supported formats are: csv, delimited, tsv, xls, blob and text. Blob - is set by default as field named 'body' of type bytes. @@ -47,7 +60,7 @@ JSON - is not supported. You must manually provide the output schema. **Delimiter:** Delimiter to use when the format is 'delimited'. This will be ignored for other formats. **Use First Row as Header:** Whether to use the first line of each file as the column headers. Supported formats are ' -text', 'csv', 'tsv', and 'delimited'. +text', 'csv', 'tsv', 'xls', and 'delimited'. **Enable Quoted Values** Whether to treat content between quotes as a value. This value will only be used if the format is 'csv', 'tsv' or 'delimited'. For example, if this is set to true, a line that looks like `1, "a, b, c"` will output diff --git a/src/main/java/io/cdap/plugin/batch/source/ftp/FTPConfig.java b/src/main/java/io/cdap/plugin/batch/source/ftp/FTPConfig.java index 3503e8d..80526ef 100644 --- a/src/main/java/io/cdap/plugin/batch/source/ftp/FTPConfig.java +++ b/src/main/java/io/cdap/plugin/batch/source/ftp/FTPConfig.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; @@ -58,6 +59,9 @@ public class FTPConfig extends PluginConfig implements FileSourceProperties { private static final Type MAP_STRING_STRING_TYPE = new TypeToken>() { }.getType(); private static final List LOCATION_PROPERTIES = Arrays.asList("type", "host", "path", "user", "password"); + private static final String NAME_SHEET = "sheet"; + private static final String NAME_SHEET_VALUE = "sheetValue"; + private static final String NAME_TERMINATE_IF_EMPTY_ROW = "terminateIfEmptyRow"; @Macro @Nullable @@ -112,13 +116,24 @@ public class FTPConfig extends PluginConfig implements FileSourceProperties { @Macro @Nullable - @Description("Whether to use first row as header. Supported formats are 'text', 'csv', 'tsv', " + + @Description("The maximum number of rows that will get investigated for automatic data type detection.") + private Long sampleSize; + + @Macro + @Nullable + @Description("A list of columns with the corresponding data types for whom the automatic data type detection gets" + + " skipped.") + private String override; + + @Macro + @Nullable + @Description("Whether to use first row as header. Supported formats are 'text', 'csv', 'tsv', 'xls', " + "'delimited'. Default value is false.") private final Boolean skipHeader; @Macro @Description("Format of the data to read. Supported formats are 'avro', 'blob', 'csv', 'delimited', 'json', " - + "'parquet', 'text', or 'tsv'. If no format is given, it will default to 'text'.") + + "'parquet', 'text', or 'tsv', 'xls'. If no format is given, it will default to 'text'.") private final String format; @Macro @@ -150,6 +165,25 @@ public class FTPConfig extends PluginConfig implements FileSourceProperties { @Description("Maximum time in milliseconds to wait for connection initialization before time out.") private final Integer connectTimeout; + @Name(NAME_SHEET) + @Macro + @Nullable + @Description("Select the sheet by name or number. Default is 'Sheet Number'.") + private String sheet; + + @Name(NAME_SHEET_VALUE) + @Macro + @Nullable + @Description("The name/number of the sheet to read from. If not specified, the first sheet will be read." + + "Sheet Number are 0 based, ie first sheet is 0.") + private String sheetValue; + + @Name(NAME_TERMINATE_IF_EMPTY_ROW) + @Macro + @Nullable + @Description("Specify whether to stop reading after encountering the first empty row. Defaults to false.") + private String terminateIfEmptyRow; + @VisibleForTesting private FTPConfig(@Nullable String referenceName, String type, String host, @Nullable Integer port, String path, String user, String password, @Nullable String fileSystemProperties, diff --git a/widgets/FTPSource-batchsource.json b/widgets/FTPSource-batchsource.json index a89ee3d..28ec22b 100644 --- a/widgets/FTPSource-batchsource.json +++ b/widgets/FTPSource-batchsource.json @@ -88,6 +88,10 @@ { "label": "tsv", "value": "tsv" + }, + { + "label": "xls", + "value": "xls" } ] } @@ -96,6 +100,36 @@ "widget-type": "get-schema", "widget-category": "plugin" }, + { + "widget-type": "number", + "label": "Sample Size", + "name": "sampleSize", + "widget-attributes": { + "default": "1000", + "minimum": "1" + } + }, + { + "widget-type": "keyvalue-dropdown", + "label": "Override", + "name": "override", + "widget-attributes": { + "key-placeholder": "Field Name", + "value-placeholder": "Data Type", + "dropdownOptions": [ + "boolean", + "bytes", + "double", + "float", + "int", + "long", + "string", + "date", + "time", + "timestamp" + ] + } + }, { "widget-type": "textbox", "label": "Delimiter", @@ -151,6 +185,42 @@ "label": "False" } } + }, + { + "widget-type": "toggle", + "label": "Terminate If Empty Row", + "name": "terminateIfEmptyRow", + "widget-attributes": { + "default": "false", + "on": { + "value": "true", + "label": "True" + }, + "off": { + "value": "false", + "label": "False" + } + } + }, + { + "widget-type": "select", + "label": "Select Sheet Using", + "name": "sheet", + "widget-attributes": { + "values": [ + "Sheet Name", + "Sheet Number" + ], + "default": "Sheet Number" + } + }, + { + "widget-type": "textbox", + "label": "Sheet Value", + "name": "sheetValue", + "widget-attributes": { + "default": "0" + } } ] }, @@ -257,13 +327,46 @@ { "name": "skipHeader", "condition": { - "expression": "format == 'delimited' || format == 'csv' || format == 'tsv'" + "expression": "format == 'delimited' || format == 'csv' || format == 'tsv' || format == 'xls'" }, "show": [ { "name": "skipHeader" } ] + }, + { + "name": "sheet", + "condition": { + "expression": "format == 'xls'" + }, + "show": [ + { + "name": "sheet" + } + ] + }, + { + "name": "sheetValue", + "condition": { + "expression": "format == 'xls'" + }, + "show": [ + { + "name": "sheetValue" + } + ] + }, + { + "name": "terminateIfEmptyRow", + "condition": { + "expression": "format == 'xls'" + }, + "show": [ + { + "name": "terminateIfEmptyRow" + } + ] } ], "jump-config": {