|
| 1 | +# Copyright 2026 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +# [START dataplex_create_data_quality_scan_global] |
| 16 | +import google.api_core.exceptions |
| 17 | +from google.cloud import dataplex_v1 |
| 18 | + |
| 19 | + |
| 20 | +def create_data_quality_scan_global( |
| 21 | + project_id: str, |
| 22 | + dataset_id: str, |
| 23 | + table_id: str, |
| 24 | + location: str, |
| 25 | + column_id_1: str, |
| 26 | + column_id_2: str, |
| 27 | +) -> None: |
| 28 | + """Creates a Dataplex Data Quality Scan using global API endpoint routing. |
| 29 | +
|
| 30 | + Args: |
| 31 | + project_id (str): GCP project ID where the scan is created. |
| 32 | + dataset_id (str): Target BigQuery dataset ID. |
| 33 | + table_id (str): Target BigQuery table ID to scan. |
| 34 | + location (str): GCP region where serverless compute runs. |
| 35 | + column_id_1 (str): Name of the first column to evaluate. |
| 36 | + column_id_2 (str): Name of the second column to evaluate. |
| 37 | + """ |
| 38 | + client = dataplex_v1.DataScanServiceClient() |
| 39 | + |
| 40 | + parent = client.common_location_path(project=project_id, location=location) |
| 41 | + |
| 42 | + # A bigquery table with at least 2 columns is assumed. |
| 43 | + bigquery_table = ( |
| 44 | + f"//bigquery.googleapis.com/projects/{project_id}" |
| 45 | + f"/datasets/{dataset_id}/tables/{table_id}" |
| 46 | + ) |
| 47 | + |
| 48 | + data_quality_spec = dataplex_v1.DataQualitySpec( |
| 49 | + rules=[ |
| 50 | + dataplex_v1.DataQualityRule( |
| 51 | + name="global-null-assertion", |
| 52 | + dimension="COMPLETENESS", |
| 53 | + description="Fails if any row contains a null value", |
| 54 | + sql_assertion=dataplex_v1.DataQualityRule.SqlAssertion( |
| 55 | + # Use ${data()} as the placeholder for the table Dataplex is scanning |
| 56 | + sql_statement=( |
| 57 | + "SELECT * FROM ${data()} " |
| 58 | + f"WHERE {column_id_1} IS NULL OR {column_id_2} IS NULL" |
| 59 | + ) |
| 60 | + ), |
| 61 | + ) |
| 62 | + ] |
| 63 | + ) |
| 64 | + |
| 65 | + data_scan = dataplex_v1.DataScan( |
| 66 | + display_name="Global Data Quality Scan", |
| 67 | + data=dataplex_v1.DataSource(resource=bigquery_table), |
| 68 | + data_quality_spec=data_quality_spec, |
| 69 | + ) |
| 70 | + |
| 71 | + request = dataplex_v1.CreateDataScanRequest(parent=parent, data_scan=data_scan) |
| 72 | + |
| 73 | + try: |
| 74 | + operation = client.create_data_scan(request=request) |
| 75 | + print(operation.result()) |
| 76 | + except google.api_core.exceptions.AlreadyExists: |
| 77 | + print("A scan with this ID already exists.") |
| 78 | + except google.api_core.exceptions.InvalidArgument as e: |
| 79 | + print(f"Your scan configuration is invalid: {e}") |
| 80 | + except google.api_core.exceptions.GoogleAPIError as e: |
| 81 | + print(f"Unexpected exception: {e}") |
| 82 | + |
| 83 | +# [END dataplex_create_data_quality_scan_global] |
0 commit comments