diff --git a/scripts/us_bls/cpi/golden_data/golden_summary_report_c_cpi_u.csv b/scripts/us_bls/cpi/golden_data/golden_summary_report_c_cpi_u.csv new file mode 100644 index 0000000000..6e7ed6ebe9 --- /dev/null +++ b/scripts/us_bls/cpi/golden_data/golden_summary_report_c_cpi_u.csv @@ -0,0 +1,2 @@ +"NumPlaces","ScalingFactors","Units","MeasurementMethods","StatVar","MinDate","observationPeriods" +"1","[]","[IndexPointBasePeriodDecember1999Equals100]","[BLSChained]","dc/x6l3mg60hpe7c","1999-12","[]" diff --git a/scripts/us_bls/cpi/golden_data/golden_summary_report_cpi_u.csv b/scripts/us_bls/cpi/golden_data/golden_summary_report_cpi_u.csv new file mode 100644 index 0000000000..8af35aad90 --- /dev/null +++ b/scripts/us_bls/cpi/golden_data/golden_summary_report_cpi_u.csv @@ -0,0 +1,2 @@ +"MinDate","NumPlaces","Units","ScalingFactors","MeasurementMethods","StatVar","observationPeriods" +"1947-01","1","[IndexPointBasePeriod1982To1984Equals100]","[]","[BLSUnchained]","dc/x6l3mg60hpe7c","[]" diff --git a/scripts/us_bls/cpi/manifest.json b/scripts/us_bls/cpi/manifest.json index c3e6780e98..adcbb998d3 100644 --- a/scripts/us_bls/cpi/manifest.json +++ b/scripts/us_bls/cpi/manifest.json @@ -27,7 +27,8 @@ "source_files": [ "input_files/*" ], - "cron_schedule": "0 3 2,16 * *" + "cron_schedule": "0 3 2,16 * *", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/us_bls/cpi/validation_config.json b/scripts/us_bls/cpi/validation_config.json new file mode 100644 index 0000000000..c069f89932 --- /dev/null +++ b/scripts/us_bls/cpi/validation_config.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { "threshold": 0.1} + }, + { + "rule_id": "check_goldens_cpi_u", + "description": "Validates cpi_u data against its golden summary report.", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report_cpi_u.csv", + "input_files": "../../input0/genmcf/summary_report.csv" + } + }, + { + "rule_id": "check_goldens_c_cpi_u", + "description": "Validates c_cpi_u data against its golden summary report.", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report_c_cpi_u.csv", + "input_files": "../../input2/genmcf/summary_report.csv" + } + } + ] +} diff --git a/statvar_imports/us_bls/bls_ces_state/golden_data/golden_summary_report.csv b/statvar_imports/us_bls/bls_ces_state/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..7919f6bbd5 --- /dev/null +++ b/statvar_imports/us_bls/bls_ces_state/golden_data/golden_summary_report.csv @@ -0,0 +1,2 @@ +"MinDate","MeasurementMethods","NumPlaces","observationPeriods","ScalingFactors","StatVar","Units" +"2020-01","[BLSSeasonallyUnadjusted, BLSSeasonallyAdjusted]","104","[P1M]","[]","Count_Person_Employed_TotalNonfarm","[]" diff --git a/statvar_imports/us_bls/bls_ces_state/manifest.json b/statvar_imports/us_bls/bls_ces_state/manifest.json index 89c8c5148e..c66edd3984 100644 --- a/statvar_imports/us_bls/bls_ces_state/manifest.json +++ b/statvar_imports/us_bls/bls_ces_state/manifest.json @@ -20,7 +20,8 @@ "source_files": [ "input_files/raw_data/*" ], - "cron_schedule": "0 06 * * 3" + "cron_schedule": "0 06 * * 3", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/us_bls/bls_ces_state/validation_config.json b/statvar_imports/us_bls/bls_ces_state/validation_config.json new file mode 100644 index 0000000000..83adba1573 --- /dev/null +++ b/statvar_imports/us_bls/bls_ces_state/validation_config.json @@ -0,0 +1,18 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { "threshold": 0.1} + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +}