From 55ca40b15c683c70d25398b5ea595da9caec9c8b Mon Sep 17 00:00:00 2001 From: Mark HM Date: Sat, 10 Oct 2026 08:15:05 +0200 Subject: [PATCH] Fail on columns the contract does not declare, when the schema says so A schema with the custom property `additionalProperties: false` (the JSON Schema term; ODCS has no field for it) gets a check that fails on fields of the data the schema does not declare, and names them. It reads only the column names, so it runs on every server read through ibis, also with --metadata-only. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + datacontract/engines/checks/check_spec.py | 7 +- datacontract/engines/checks/create_checks.py | 23 ++++ datacontract/engines/checks/dimensions.py | 1 + .../engines/ibis/ibis_check_execute.py | 24 ++++ docs/docs/schema.md | 16 ++- tests/test_test_additional_fields.py | 104 ++++++++++++++++++ 7 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 tests/test_test_additional_fields.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f10ec86..e75f6751d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `datacontract lint --all-errors`: unknown fields are warnings instead of errors, unless a custom `--schema` rejects them ### Added +- `datacontract test`: a schema with the custom property `additionalProperties: false` fails on columns it does not declare - `datacontract edit`: enable the editor's AI assistant via `DATACONTRACT_EDITOR_AI_*` environment variables (endpoint, API key, model, provider, auth header) - `datacontract test`: check constraints and quality rules on nested properties on Athena and Trino servers diff --git a/datacontract/engines/checks/check_spec.py b/datacontract/engines/checks/check_spec.py index 18f49b38f..7ff88c9d4 100644 --- a/datacontract/engines/checks/check_spec.py +++ b/datacontract/engines/checks/check_spec.py @@ -32,6 +32,7 @@ class MetricType(str, Enum): FIELD_TYPE = "field_type" FIELD_PHYSICAL_TYPE = "field_physical_type" FIELD_NESTED_TYPE = "field_nested_type" + FIELD_NAMES = "field_names" FRESHNESS = "freshness" RETENTION = "retention" CUSTOM_SQL = "custom_sql" @@ -46,6 +47,7 @@ class MetricType(str, Enum): MetricType.FIELD_TYPE, MetricType.FIELD_PHYSICAL_TYPE, MetricType.FIELD_NESTED_TYPE, + MetricType.FIELD_NAMES, } @@ -159,7 +161,8 @@ class CheckSpec: expected_schema_property: Optional["SchemaProperty"] = None # FIELD_TYPE: structural comparison expected_physical_type: Optional[str] = None # FIELD_PHYSICAL_TYPE: contract physicalType - columns: Optional[List[str]] = None # DUPLICATE_COUNT across multiple columns; MISSING_REFERENCE_COUNT keys + # DUPLICATE_COUNT across multiple columns; MISSING_REFERENCE_COUNT keys; FIELD_NAMES the declared fields + columns: Optional[List[str]] = None referenced_model: Optional[str] = None # MISSING_REFERENCE_COUNT referenced_columns: Optional[List[str]] = None # MISSING_REFERENCE_COUNT, pairwise with `columns` @@ -170,7 +173,7 @@ class CheckSpec: seconds: Optional[int] = None # FRESHNESS / RETENTION threshold in seconds - uses_raw_view: bool = False # FIELD_PRESENT against the duckdb {model}__raw__ view + uses_raw_view: bool = False # FIELD_PRESENT / FIELD_NAMES against the duckdb {model}__raw__ view # Preset result/reason for checks that are not executed (UNSUPPORTED). preset_result: Optional[str] = None diff --git a/datacontract/engines/checks/create_checks.py b/datacontract/engines/checks/create_checks.py index f60ffb1ec..f7d392c5c 100644 --- a/datacontract/engines/checks/create_checks.py +++ b/datacontract/engines/checks/create_checks.py @@ -63,6 +63,15 @@ def _get_logical_type_option(prop: SchemaProperty, key: str): return prop.logicalTypeOptions.get(key) +def _no_additional_properties(schema_object: SchemaObject) -> bool: + """Whether the schema declares, as in JSON Schema, that the data has no fields beyond its properties. + + ODCS has no field for this, so it is the schema's custom property `additionalProperties: false`. + """ + value = next((c.value for c in schema_object.customProperties or [] if c.property == "additionalProperties"), None) + return value is False or str(value).strip().lower() == "false" + + def is_check_types(server: Optional[Server]) -> bool: """Type checks only make sense where the data source carries real types.""" if server is None: @@ -625,6 +634,20 @@ def _to_schema_checks( check.preset_result = "warning" check.preset_reason = reason + if _no_additional_properties(schema_object): + checks.append( + CheckSpec( + key=f"{model}__no_additional_fields", + category="schema", + type="model_no_additional_fields", + name=f"Check that {model} has no fields the contract does not declare", + model=model, + metric=MetricType.FIELD_NAMES, + columns=[prop.physicalName or prop.name for prop in properties], + uses_raw_view=uses_raw_view, + ) + ) + if primary_key_is_composite: primary_key_fields = [prop.physicalName or prop.name for prop in primary_key_props] checks.append( diff --git a/datacontract/engines/checks/dimensions.py b/datacontract/engines/checks/dimensions.py index ec3204a46..d8a5c1eb6 100644 --- a/datacontract/engines/checks/dimensions.py +++ b/datacontract/engines/checks/dimensions.py @@ -29,6 +29,7 @@ # (the SAP HANA engine reports the missing table as a check of its own) "model_exists": "conformity", "field_is_present": "conformity", + "model_no_additional_fields": "conformity", "field_type": "conformity", "field_physical_type": "conformity", "field_nested_type": "conformity", diff --git a/datacontract/engines/ibis/ibis_check_execute.py b/datacontract/engines/ibis/ibis_check_execute.py index f8e6cce54..5a035c311 100644 --- a/datacontract/engines/ibis/ibis_check_execute.py +++ b/datacontract/engines/ibis/ibis_check_execute.py @@ -94,6 +94,8 @@ def _describe(spec: CheckSpec) -> str: return f"nested_types({spec.field}) match the contract" if spec.metric == MetricType.FIELD_PRESENT: return f"present({spec.field})" + if spec.metric == MetricType.FIELD_NAMES: + return f"fields({spec.model}) in ({', '.join(spec.columns or [])})" if spec.threshold is not None: target = spec.field or spec.model return f"{spec.metric.value}({target}) {spec.threshold.describe()}" @@ -336,6 +338,8 @@ def _invalid(c, _t=t, _dtype=dtype, _spec=spec, _flag=unconstrained): _run_missing_reference(run, con, server, t, columns, spec, schema_name) elif spec.metric == MetricType.FIELD_PRESENT: _run_present(run, con, model, columns, schema, spec) + elif spec.metric == MetricType.FIELD_NAMES: + _run_field_names(run, con, model, t, spec) elif spec.metric == MetricType.FIELD_TYPE: _run_type(run, schema, columns, spec, structured_types, native_types) elif spec.metric == MetricType.FIELD_PHYSICAL_TYPE: @@ -891,6 +895,26 @@ def _run_present(run: Run, con, model: str, columns, schema, spec: CheckSpec): ) +def _run_field_names(run: Run, con, model: str, t, spec: CheckSpec): + """The fields of the data that the contract does not declare; names compare case-insensitively.""" + table = t + if spec.uses_raw_view: + try: + table = _resolve_table(con, f"{model}__raw__") + except Exception: + pass + _set_impl(run, spec.key, _describe(spec), "introspection") + declared = {name.lower() for name in spec.columns or []} + additional = [name for name in table.columns if name.lower() not in declared] + _set_diagnostics(run, spec.key, _diag(metric="field_names", additional_fields=additional)) + set_result( + run, + spec.key, + ResultEnum.failed if additional else ResultEnum.passed, + f"Fields not in the contract: {', '.join(additional)}" if additional else None, + ) + + def _run_type( run: Run, schema, diff --git a/docs/docs/schema.md b/docs/docs/schema.md index 62462df8e..b979ab3b0 100644 --- a/docs/docs/schema.md +++ b/docs/docs/schema.md @@ -33,6 +33,7 @@ datacontract test --checks properties datacontract.yaml | `logicalTypeOptions.minItems` / `maxItems` | array property | Number of elements within bounds | | `logicalTypeOptions.uniqueItems` | array property | Elements of the array are distinct | | `quality` | schema, property | See [Define your Quality Rules](./quality-rules/index.md) | +| `customProperties` `additionalProperties: false` | schema | The data has no columns the schema does not declare | A contract that uses all of them: @@ -91,6 +92,19 @@ schema: physicalName: order_total # the real column ``` +Columns the contract does not declare are allowed, as many contracts describe only part of a table. To fail on them, declare `additionalProperties: false` (the JSON Schema term) as a custom property of the schema, since ODCS has no field for it. The check names the undeclared columns; names compare case-insensitively. + +```yaml +schema: + - name: orders + customProperties: + - property: additionalProperties + value: false + properties: + - name: order_id + - name: total +``` + ## Types A property can declare a portable `logicalType`, a native `physicalType`, or both. Which one is checked depends on the backend: @@ -218,7 +232,7 @@ These are common sources of confusion. They are valid ODCS and appear in exports - **`isNullable`** — the CLI reads `required`, not `isNullable`. Write `required: true` to assert that a column has no nulls. - **`logicalTypeOptions.format`** — never enforced, on any property. On a `string` property (`email`, `uuid`, `uri`, …) use `pattern` for an enforceable equivalent. On a `date`, `timestamp` or `time` property `format` holds a date pattern such as `yyyy-MM-dd`, and `pattern` is *not* an equivalent there: by the time a check runs, the column has already been parsed as a date, so there is no string left to match. Such a column is validated as a date, in whatever format the server stores it. -- **Descriptive attributes** — `description`, `businessName`, `examples`, `tags`, `classification`, `criticalDataElement`, `transformSourceObjects`, and `customProperties`. `authoritativeDefinitions` generates no check either, but it *is* resolved and inlined before the checks are built — see [Link your Semantics](./semantics.md). +- **Descriptive attributes** — `description`, `businessName`, `examples`, `tags`, `classification`, `criticalDataElement`, `transformSourceObjects`, and `customProperties` (except `additionalProperties` on a schema). `authoritativeDefinitions` generates no check either, but it *is* resolved and inlined before the checks are built — see [Link your Semantics](./semantics.md). - **Schema-level attributes** other than `name`, `physicalName`, `properties`, and `quality`. - **Constraints and quality rules on nested properties, on some servers** — `required`, `unique`, `pattern`, `enum`, the other `logicalTypeOptions` and `quality` rules below the top level are checked on the `dataframe`, `databricks`, `trino` and `athena` servers and on every server read through DuckDB (`local`, `s3`, `gcs`, `azure`, `duckdb`, `iceberg`, `kafka`). On every other server, nested properties are only type-checked, and each of their constraints and quality rules is reported as a warning. On SAP HANA, only their quality rules are reported as warnings; their constraints are not checked. diff --git a/tests/test_test_additional_fields.py b/tests/test_test_additional_fields.py new file mode 100644 index 000000000..2ae505dc8 --- /dev/null +++ b/tests/test_test_additional_fields.py @@ -0,0 +1,104 @@ +"""A schema with the custom property `additionalProperties: false` fails on data with fields it does not declare.""" + +import pytest + +from datacontract.data_contract import DataContract +from datacontract.model.run import ResultEnum + +PROPERTIES = """ - name: order_id + logicalType: string + - name: quantity + logicalType: integer""" + + +def _contract( + path, file_format, switch="\n customProperties:\n - property: additionalProperties\n value: false" +): + return f"""apiVersion: v3.2.0 +kind: DataContract +id: orders +name: orders +version: 1.0.0 +status: active +servers: + - server: local + type: local + format: {file_format} + path: {path} +schema: + - name: orders{switch} + properties: +{PROPERTIES} +""" + + +def _csv(tmp_path, header): + data = tmp_path / "orders.csv" + data.write_text(f"{header}\nA1|3\n" if header.count("|") == 1 else f"{header}\nA1|3|x\n") + return data + + +def _check(run): + return next(c for c in run.checks if c.type == "model_no_additional_fields") + + +def test_a_field_the_contract_does_not_declare_fails(tmp_path): + run = DataContract(data_contract_str=_contract(_csv(tmp_path, "order_id|quantity|note"), "csv")).test() + print(run.pretty()) + check = _check(run) + assert check.result == ResultEnum.failed + assert check.reason == "Fields not in the contract: note" + assert check.diagnostics["additional_fields"] == ["note"] + assert [c.type for c in run.checks if c.result != ResultEnum.passed] == ["model_no_additional_fields"] + + +def test_exactly_the_declared_fields_pass(tmp_path): + run = DataContract(data_contract_str=_contract(_csv(tmp_path, "order_id|quantity"), "csv")).test() + assert _check(run).result == ResultEnum.passed + assert run.result == ResultEnum.passed + + +def test_field_names_compare_case_insensitively(tmp_path): + run = DataContract(data_contract_str=_contract(_csv(tmp_path, "ORDER_ID|Quantity"), "csv")).test() + assert _check(run).result == ResultEnum.passed + + +def test_without_the_custom_property_additional_fields_are_allowed(tmp_path): + run = DataContract(data_contract_str=_contract(_csv(tmp_path, "order_id|quantity|note"), "csv", switch="")).test() + assert not [c for c in run.checks if c.type == "model_no_additional_fields"] + assert run.result == ResultEnum.passed + + +@pytest.mark.parametrize("value", ["'false'", "False"]) +def test_the_value_may_be_text(tmp_path, value): + switch = f"\n customProperties:\n - property: additionalProperties\n value: {value}" + run = DataContract(data_contract_str=_contract(_csv(tmp_path, "order_id|quantity|note"), "csv", switch)).test() + assert _check(run).result == ResultEnum.failed + + +def test_a_json_record_with_an_undeclared_key_fails(tmp_path): + data = tmp_path / "orders.json" + data.write_text('[{"order_id": "A1", "quantity": 3, "note": "x"}]') + run = DataContract(data_contract_str=_contract(data, "json")).test() + assert _check(run).reason == "Fields not in the contract: note" + + +def test_the_check_reads_no_rows(tmp_path): + contract = _contract(_csv(tmp_path, "order_id|quantity|note"), "csv") + run = DataContract(data_contract_str=contract, metadata_only=True).test() + assert _check(run).result == ResultEnum.failed + + +def test_a_database_table_with_an_undeclared_column_fails(tmp_path): + import duckdb + + database = tmp_path / "orders.duckdb" + with duckdb.connect(str(database)) as con: + con.sql("CREATE TABLE orders (order_id VARCHAR, quantity INTEGER, note VARCHAR)") + con.sql("INSERT INTO orders VALUES ('A1', 3, 'x')") + contract = _contract(tmp_path, "csv").replace( + f" type: local\n format: csv\n path: {tmp_path}", f" type: duckdb\n database: {database}" + ) + run = DataContract(data_contract_str=contract).test() + print(run.pretty()) + assert _check(run).reason == "Fields not in the contract: note"