From 25f06063d68ca4bf26831064bba9a46298583d65 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 4 Mar 2024 09:26:55 +0530 Subject: [PATCH] Added ArgumentsColumnsList Exists Validation asdsadasd --- .../action/BigQueryArgumentSetterConfig.java | 20 +++++++ .../BigQueryArgumentSetterConfigTest.java | 54 +++++++++++++++++++ widgets/BigQueryArgumentSetter-action.json | 2 +- 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfig.java b/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfig.java index 7c882769dd..d0ffb827e2 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfig.java +++ b/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfig.java @@ -198,6 +198,7 @@ public QueryJobConfiguration getQueryJobConfiguration(FailureCollector collector Map argumentConditionFields = extractArgumentsFields(fields, argumentConditionMap); checkIfArgumentsColumnsExitsInSource(argumentConditionMap, argumentConditionFields); + checkIfArgumentsColumnsListExistsInSource(getArgumentsColumnsList(), fields, collector); String selectClause = getSelectClause(); String whereCondition = getWhereCondition(argumentConditionMap.keySet()); @@ -228,6 +229,25 @@ private void checkIfArgumentsColumnsExitsInSource(Map argumentCo nonExistingColumnNames)); } + static void checkIfArgumentsColumnsListExistsInSource( + List argumentsColumnsList, FieldList fields, FailureCollector collector) { + Set fieldsNames = fields.stream().map(Field::getName).collect(Collectors.toSet()); + List nonExistingColumns = argumentsColumnsList.stream() + .filter(columnName -> !fieldsNames.contains(columnName)) + .collect(Collectors.toList()); + if (!nonExistingColumns.isEmpty()) { + String formattedNonExistingColumnsList = String.join(" ,", nonExistingColumns); + String[] oneManyColumn = {"Column", "Columns"}; + String[] oneManyDo = {"does", "do"}; + int oneManyIndex = nonExistingColumns.size() == 1 ? 0 : 1; + collector.addFailure( + String.format("%s: \"%s\" %s not exist in table. Argument columns must exist in table.", + oneManyColumn[oneManyIndex], formattedNonExistingColumnsList, oneManyDo[oneManyIndex]), + "Ensure correct column names are provided.") + .withConfigProperty(NAME_ARGUMENTS_COLUMNS); + } + } + private Map extractArgumentsFields( FieldList fields, Map argumentConditionKeyPair) { return fields.stream() diff --git a/src/test/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfigTest.java b/src/test/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfigTest.java index 6dea7557f2..caf8dd988c 100644 --- a/src/test/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfigTest.java +++ b/src/test/java/io/cdap/plugin/gcp/bigquery/action/BigQueryArgumentSetterConfigTest.java @@ -16,13 +16,20 @@ package io.cdap.plugin.gcp.bigquery.action; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.common.collect.ImmutableList; import io.cdap.cdap.etl.api.validation.CauseAttributes; import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import java.util.List; + public class BigQueryArgumentSetterConfigTest { private static final String VALID_DATASET = "dataset"; @@ -30,6 +37,12 @@ public class BigQueryArgumentSetterConfigTest { private static final String VALID_TABLE = "table"; private static final String VALID_ARGUMENT_SELECTION_CONDITIONS = "feed=10;id=0"; private static final String VALID_ARGUMENT_COLUMN = "name"; + private MockFailureCollector mockFailureCollector; + + @Before + public void setUp() { + mockFailureCollector = new MockFailureCollector(); + } @Test public void testValidateMissingArgumentSelectionConditions() { @@ -43,6 +56,47 @@ public void testValidateMissingArgumentColumns() { validateConfigValidationFail(config, BigQueryArgumentSetterConfig.NAME_ARGUMENTS_COLUMNS); } + @Test + public void testCheckIfArgumentsColumnsListExistsInSourceMatchOne() { + List argumentsColumnsList = ImmutableList.of("name"); + FieldList fields = FieldList.of(Field.newBuilder("name", LegacySQLTypeName.STRING).build()); + BigQueryArgumentSetterConfig.checkIfArgumentsColumnsListExistsInSource(argumentsColumnsList, fields, + mockFailureCollector); + Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); + } + + @Test + public void testCheckIfArgumentsColumnsListExistsInSourceMatchSome() { + List argumentsColumnsList = ImmutableList.of("name"); + FieldList fields = FieldList.of(Field.newBuilder("name", LegacySQLTypeName.STRING).build(), + Field.newBuilder("age", LegacySQLTypeName.INTEGER).build()); + BigQueryArgumentSetterConfig.checkIfArgumentsColumnsListExistsInSource(argumentsColumnsList, fields, + mockFailureCollector); + Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); + } + + @Test + public void testCheckIfArgumentsColumnsListExistsInSourceFailedMatchOne() { + List argumentsColumnsList = ImmutableList.of("name"); + FieldList fields = FieldList.of(Field.newBuilder("age", LegacySQLTypeName.INTEGER).build()); + BigQueryArgumentSetterConfig.checkIfArgumentsColumnsListExistsInSource(argumentsColumnsList, fields, + mockFailureCollector); + Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); + Assert.assertEquals("Column: \"name\" does not exist in table. Argument columns must exist in table.", + mockFailureCollector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testCheckIfArgumentsColumnsListExistsInSourceFailedMatchSome() { + List argumentsColumnsList = ImmutableList.of("name", "city"); + FieldList fields = FieldList.of(Field.newBuilder("age", LegacySQLTypeName.INTEGER).build()); + BigQueryArgumentSetterConfig.checkIfArgumentsColumnsListExistsInSource(argumentsColumnsList, fields, + mockFailureCollector); + Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); + Assert.assertEquals("Columns: \"name ,city\" do not exist in table. Argument columns must exist in table.", + mockFailureCollector.getValidationFailures().get(0).getMessage()); + } + private static BigQueryArgumentSetterConfig.Builder getBuilder() { return BigQueryArgumentSetterConfig.builder() .setDatasetProject(VALID_DATASET_PROJECT) diff --git a/widgets/BigQueryArgumentSetter-action.json b/widgets/BigQueryArgumentSetter-action.json index 6c905a3d23..bf20dbde42 100644 --- a/widgets/BigQueryArgumentSetter-action.json +++ b/widgets/BigQueryArgumentSetter-action.json @@ -72,7 +72,7 @@ "widget-type": "csv", "widget-attributes" : { "delimiter": ",", - "placeholder": "The name of the column that contains the arguments for this run." + "value-placeholder": "The name of the column that contains the arguments for this run." } } ]