diff --git a/scripts/census_county_business_patterns/manifest.json b/scripts/census_county_business_patterns/manifest.json index bce5e29199..f3c1f5044b 100644 --- a/scripts/census_county_business_patterns/manifest.json +++ b/scripts/census_county_business_patterns/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "CensusCountyBusinessPatterns.tmcf", - "cleaned_csv": "gcs_output/output/*.csv" + "cleaned_csv": "gcs_output/output/*.csv", + "node_mcf": "gcs_output/output/*.mcf" } ], "cron_schedule": "0 9 5,25 * *", diff --git a/scripts/covid19_india/cases_count_states_data/manifest.json b/scripts/covid19_india/cases_count_states_data/manifest.json index 18aec0bfa7..b5e2e5ade4 100644 --- a/scripts/covid19_india/cases_count_states_data/manifest.json +++ b/scripts/covid19_india/cases_count_states_data/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "COVID19_cases_indian_states.tmcf", - "cleaned_csv": "COVID19_cases_indian_states.csv" + "cleaned_csv": "COVID19_cases_indian_states.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 5 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/covid19_india/medical_tests_in_data/manifest.json b/scripts/covid19_india/medical_tests_in_data/manifest.json index ad829b771e..f6edd343fd 100644 --- a/scripts/covid19_india/medical_tests_in_data/manifest.json +++ b/scripts/covid19_india/medical_tests_in_data/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "COVID19_tests_india.tmcf", - "cleaned_csv": "COVID19_tests_india.csv" + "cleaned_csv": "COVID19_tests_india.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 6 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/covid_tracking_project/historic_us_data/manifest.json b/scripts/covid_tracking_project/historic_us_data/manifest.json index 89eb263466..e2327f8943 100644 --- a/scripts/covid_tracking_project/historic_us_data/manifest.json +++ b/scripts/covid_tracking_project/historic_us_data/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "COVIDTracking_US.tmcf", - "cleaned_csv": "COVIDTracking_US.csv" + "cleaned_csv": "COVIDTracking_US.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "30 3 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/health_determinants/manifest.json b/scripts/eurostat/health_determinants/manifest.json index 3dc7907d90..e70fa181ea 100644 --- a/scripts/eurostat/health_determinants/manifest.json +++ b/scripts/eurostat/health_determinants/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "alcohol_consumption/output/eurostat_population_alcoholconsumption.tmcf", - "cleaned_csv": "alcohol_consumption/output/eurostat_population_alcoholconsumption.csv" + "cleaned_csv": "alcohol_consumption/output/eurostat_population_alcoholconsumption.csv", + "node_mcf": "alcohol_consumption/output/*.mcf" } ], "source_files": [ @@ -39,7 +40,8 @@ "import_inputs": [ { "template_mcf": "bmi/output/eurostat_population_bmi.tmcf", - "cleaned_csv": "bmi/output/eurostat_population_bmi.csv" + "cleaned_csv": "bmi/output/eurostat_population_bmi.csv", + "node_mcf": "bmi/output/*.mcf" } ], "source_files": [ @@ -65,7 +67,8 @@ "import_inputs": [ { "template_mcf": "fruits_vegetables/output/eurostat_population_fruits_vegetables.tmcf", - "cleaned_csv": "fruits_vegetables/output/eurostat_population_fruits_vegetables.csv" + "cleaned_csv": "fruits_vegetables/output/eurostat_population_fruits_vegetables.csv", + "node_mcf": "fruits_vegetables/output/*.mcf" } ], "source_files": [ @@ -91,7 +94,8 @@ "import_inputs": [ { "template_mcf": "physical_activity/output/eurostat_population_physicalactivity.tmcf", - "cleaned_csv": "physical_activity/output/eurostat_population_physicalactivity.csv" + "cleaned_csv": "physical_activity/output/eurostat_population_physicalactivity.csv", + "node_mcf": "physical_activity/output/*.mcf" } ], "source_files": [ @@ -117,7 +121,8 @@ "import_inputs": [ { "template_mcf": "social_environment/output/eurostat_population_social_environment.tmcf", - "cleaned_csv": "social_environment/output/eurostat_population_social_environment.csv" + "cleaned_csv": "social_environment/output/eurostat_population_social_environment.csv", + "node_mcf": "social_environment/output/*.mcf" } ], "source_files": [ @@ -143,7 +148,8 @@ "import_inputs": [ { "template_mcf": "tobacco_consumption/output/eurostat_population_tobaccoconsumption.tmcf", - "cleaned_csv": "tobacco_consumption/output/eurostat_population_tobaccoconsumption.csv" + "cleaned_csv": "tobacco_consumption/output/eurostat_population_tobaccoconsumption.csv", + "node_mcf": "tobacco_consumption/output/*.mcf" } ], "source_files": [ @@ -157,4 +163,4 @@ "cron_schedule": "0 3 2 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json index b5820425f0..80ab7c0fe0 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "EurostatNUTS3_BirthDeathMigration.tmcf", - "cleaned_csv": "EurostatNUTS3_BirthDeathMigration.csv" + "cleaned_csv": "EurostatNUTS3_BirthDeathMigration.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "00 02 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json index a140431d7b..cba666262c 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "Eurostats_NUTS2_Edat.tmcf", - "cleaned_csv": "Eurostats_NUTS2_Edat.csv" + "cleaned_csv": "Eurostats_NUTS2_Edat.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 02 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json index 60ac5e00a8..f87087c77b 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "Eurostats_NUTS2_Enrollment.tmcf", - "cleaned_csv": "Eurostats_NUTS2_Enrollment.csv" + "cleaned_csv": "Eurostats_NUTS2_Enrollment.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "00 03 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/employed_per_sector/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/employed_per_sector/manifest.json index 27b6892e0a..b180757848 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/employed_per_sector/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/employed_per_sector/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "Eurostats_NUTS3_Empers.tmcf", - "cleaned_csv": "Eurostats_NUTS3_Empers.csv" + "cleaned_csv": "Eurostats_NUTS3_Empers.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 03 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/fertility_rate_mother_age/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/fertility_rate_mother_age/manifest.json index 76cadc3646..1301a1aa3a 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/fertility_rate_mother_age/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/fertility_rate_mother_age/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "Eurostats_NUTS3_FRate_Age.tmcf", - "cleaned_csv": "Eurostats_NUTS3_FRate_Age.csv" + "cleaned_csv": "Eurostats_NUTS3_FRate_Age.csv", + "node_mcf": "*.mcf" } ], "source_files": [ diff --git a/scripts/eurostat/regional_statistics_by_nuts/gdp/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/gdp/manifest.json index a1b3039cb6..44c45139d9 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/gdp/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/gdp/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "eurostat_gdp.tmcf", - "cleaned_csv": "eurostat_gdp.csv" + "cleaned_csv": "eurostat_gdp.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 04 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/life_expectancy/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/life_expectancy/manifest.json index f387348845..c6a69959a3 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/life_expectancy/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/life_expectancy/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "demo_r_mlifexp.tmcf", - "cleaned_csv": "demo_r_mlifexp_cleaned.csv" + "cleaned_csv": "demo_r_mlifexp_cleaned.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "00 05 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/population_density/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/population_density/manifest.json index ca1b99bb35..761e86e070 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/population_density/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/population_density/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "PopulationDensity_Eurostat_NUTS3.tmcf", - "cleaned_csv": "PopulationDensity_Eurostat_NUTS3.csv" + "cleaned_csv": "PopulationDensity_Eurostat_NUTS3.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 05 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/fbi/hate_crime/aggregations/manifest.json b/scripts/fbi/hate_crime/aggregations/manifest.json index f7775090a3..ff485c8f77 100644 --- a/scripts/fbi/hate_crime/aggregations/manifest.json +++ b/scripts/fbi/hate_crime/aggregations/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "../aggregations.tmcf", - "cleaned_csv": "aggregation.csv" + "cleaned_csv": "aggregation.csv", + "node_mcf": "*.mcf" } ], "source_files": [ diff --git a/scripts/fbi/hate_crime/manifest.json b/scripts/fbi/hate_crime/manifest.json index 6219706d8e..696232fa53 100644 --- a/scripts/fbi/hate_crime/manifest.json +++ b/scripts/fbi/hate_crime/manifest.json @@ -28,23 +28,28 @@ "import_inputs": [ { "template_mcf": "tables1-10/t1tot10.tmcf", - "cleaned_csv": "tables1-10/t1tot10_combined.csv" + "cleaned_csv": "tables1-10/t1tot10_combined.csv", + "node_mcf": "tables1-10/*.mcf" }, { "template_mcf": "table11/table11_output.tmcf", - "cleaned_csv": "table11/table11_output.csv" + "cleaned_csv": "table11/table11_output.csv", + "node_mcf": "table11/*.mcf" }, { "template_mcf": "table12/table12_output.tmcf", - "cleaned_csv": "table12/table12_output.csv" + "cleaned_csv": "table12/table12_output.csv", + "node_mcf": "table12/*.mcf" }, { "template_mcf": "table13/table13_output.tmcf", - "cleaned_csv": "table13/table13_output.csv" + "cleaned_csv": "table13/table13_output.csv", + "node_mcf": "table13/*.mcf" }, { "template_mcf": "table14/table14_output.tmcf", - "cleaned_csv": "table14/table14_output.csv" + "cleaned_csv": "table14/table14_output.csv", + "node_mcf": "table14/*.mcf" } ], "source_files": [ diff --git a/scripts/fires/firms/manifest.json b/scripts/fires/firms/manifest.json index bda7e317ea..261ed94011 100644 --- a/scripts/fires/firms/manifest.json +++ b/scripts/fires/firms/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "", - "cleaned_csv": "" + "cleaned_csv": "", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 5 1 * *", @@ -25,4 +26,4 @@ } } ] -} \ No newline at end of file +} diff --git a/scripts/fires/manifest.json b/scripts/fires/manifest.json index 65e644b292..8fbf29c789 100644 --- a/scripts/fires/manifest.json +++ b/scripts/fires/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "wfigs.tmcf", - "cleaned_csv": "final_processed_data.csv" + "cleaned_csv": "final_processed_data.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 6 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/fires/perimeter/manifest.json b/scripts/fires/perimeter/manifest.json index b264e4fcc3..b9439f224f 100644 --- a/scripts/fires/perimeter/manifest.json +++ b/scripts/fires/perimeter/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "fire_geojson.tmcf", - "cleaned_csv": "perimeter_data.csv" + "cleaned_csv": "perimeter_data.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 6 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/floods/manifest.json b/scripts/floods/manifest.json index 063780bdd1..20cfbfac7f 100644 --- a/scripts/floods/manifest.json +++ b/scripts/floods/manifest.json @@ -13,10 +13,11 @@ "import_inputs": [ { "template_mcf": "", - "cleaned_csv": "" + "cleaned_csv": "", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 5 1 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/google_covid/mobility/manifest.json b/scripts/google_covid/mobility/manifest.json index 8b963308ac..2417f806ac 100644 --- a/scripts/google_covid/mobility/manifest.json +++ b/scripts/google_covid/mobility/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "covid_mobility.tmcf", - "cleaned_csv": "covid_mobility.csv" + "cleaned_csv": "covid_mobility.csv", + "node_mcf": "*.mcf" } ], "cron_schedule": "30 2 10 * *", diff --git a/scripts/india_rbi/below_poverty_line/manifest.json b/scripts/india_rbi/below_poverty_line/manifest.json index 3627381bce..a65adbe49e 100644 --- a/scripts/india_rbi/below_poverty_line/manifest.json +++ b/scripts/india_rbi/below_poverty_line/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "BelowPovertyLine_india.tmcf", - "cleaned_csv": "BelowPovertyLine_india.csv" + "cleaned_csv": "BelowPovertyLine_india.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -21,4 +22,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/scripts/noaa/gpcc_spi/manifest.json b/scripts/noaa/gpcc_spi/manifest.json index 77923a6058..01ba39fc74 100644 --- a/scripts/noaa/gpcc_spi/manifest.json +++ b/scripts/noaa/gpcc_spi/manifest.json @@ -16,26 +16,31 @@ "import_inputs": [ { "template_mcf": "gpcc_spi.tmcf", - "cleaned_csv": "output_files/*.csv" + "cleaned_csv": "output_files/*.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "gpcc_spi_aggregation.tmcf", - "cleaned_csv": "output_files/aggregations/*.csv" + "cleaned_csv": "output_files/aggregations/*.csv", + "node_mcf": "output_files/aggregations/*.mcf" }, { "template_mcf": "drought_spi_9m_polygon_place_svobs.tmcf", - "cleaned_csv": "output_files/events_spi_9m_polygon/place_svobs/drought_spi_9m_polygon_place_svobs.csv" + "cleaned_csv": "output_files/events_spi_9m_polygon/place_svobs/drought_spi_9m_polygon_place_svobs.csv", + "node_mcf": "output_files/events_spi_9m_polygon/place_svobs/*.mcf" }, { "template_mcf": "drought_spi_9m_polygon_events.tmcf", - "cleaned_csv": "output_files/events_spi_9m_polygon/events/drought_spi_9m_polygon_events.csv" + "cleaned_csv": "output_files/events_spi_9m_polygon/events/drought_spi_9m_polygon_events.csv", + "node_mcf": "output_files/events_spi_9m_polygon/events/*.mcf" }, { "template_mcf": "drought_spi_9m_polygon_svobs.tmcf", - "cleaned_csv": "output_files/events_spi_9m_polygon/event_svobs/drought_spi_9m_polygon_svobs.csv" + "cleaned_csv": "output_files/events_spi_9m_polygon/event_svobs/drought_spi_9m_polygon_svobs.csv", + "node_mcf": "output_files/events_spi_9m_polygon/event_svobs/*.mcf" } ], "cron_schedule": "0 9 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/noaa_gfs/manifest.json b/scripts/noaa_gfs/manifest.json index 009bdc4258..f99ed91ee4 100644 --- a/scripts/noaa_gfs/manifest.json +++ b/scripts/noaa_gfs/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "./noaa_gfs_output.tmcf", - "cleaned_csv": "output/*/gfs.t*.csv" + "cleaned_csv": "output/*/gfs.t*.csv", + "node_mcf": "output/*/*.mcf" } ], "source_files": [ diff --git a/scripts/oecd/regional_demography/deaths/manifest.json b/scripts/oecd/regional_demography/deaths/manifest.json index 3ad3167df4..b1a7fcb5b6 100644 --- a/scripts/oecd/regional_demography/deaths/manifest.json +++ b/scripts/oecd/regional_demography/deaths/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "OECD_deaths_cleaned.tmcf", - "cleaned_csv": "OECD_deaths_cleaned.csv" + "cleaned_csv": "OECD_deaths_cleaned.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 7 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/oecd/regional_demography/life_expectancy/manifest.json b/scripts/oecd/regional_demography/life_expectancy/manifest.json index 185e880a88..e5fabdfc21 100644 --- a/scripts/oecd/regional_demography/life_expectancy/manifest.json +++ b/scripts/oecd/regional_demography/life_expectancy/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "OECD_life_expectancy_cleaned.tmcf", - "cleaned_csv": "OECD_life_expectancy_cleaned.csv" + "cleaned_csv": "OECD_life_expectancy_cleaned.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 8 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/oecd/regional_demography/pop_density/manifest.json b/scripts/oecd/regional_demography/pop_density/manifest.json index d235a25ffe..4c9081080c 100644 --- a/scripts/oecd/regional_demography/pop_density/manifest.json +++ b/scripts/oecd/regional_demography/pop_density/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "OECD_pop_density_cleaned.tmcf", - "cleaned_csv": "OECD_pop_density_cleaned.csv" + "cleaned_csv": "OECD_pop_density_cleaned.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 9 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/oecd/regional_demography/population/manifest.json b/scripts/oecd/regional_demography/population/manifest.json index 7244bc81e5..6848752503 100644 --- a/scripts/oecd/regional_demography/population/manifest.json +++ b/scripts/oecd/regional_demography/population/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "OECD_population_cleaned.tmcf", - "cleaned_csv": "OECD_population_cleaned.csv" + "cleaned_csv": "OECD_population_cleaned.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 10 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/ourworldindata/covid19/manifest.json b/scripts/ourworldindata/covid19/manifest.json index e8d0fc31a8..cb47770c28 100644 --- a/scripts/ourworldindata/covid19/manifest.json +++ b/scripts/ourworldindata/covid19/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "OurWorldInData_Covid19.tmcf", - "cleaned_csv": "OurWorldInData_Covid19.csv" + "cleaned_csv": "OurWorldInData_Covid19.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 2 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/pcm-dpc/covid-19/manifest.json b/scripts/pcm-dpc/covid-19/manifest.json index 390c67064e..03b2c25cb0 100644 --- a/scripts/pcm-dpc/covid-19/manifest.json +++ b/scripts/pcm-dpc/covid-19/manifest.json @@ -18,11 +18,13 @@ }, { "template_mcf": "dpc-covid19-ita-province.tmcf", - "cleaned_csv": "dpc-covid19-ita-province.csv" + "cleaned_csv": "dpc-covid19-ita-province.csv", + "node_mcf": "*.mcf" }, { "template_mcf": "dpc-covid19-ita-national-trend.tmcf", - "cleaned_csv": "dpc-covid19-ita-national-trend.csv" + "cleaned_csv": "dpc-covid19-ita-national-trend.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -31,4 +33,4 @@ "cron_schedule": "0 18 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/un/energy/manifest.json b/scripts/un/energy/manifest.json index 65e013c19b..7ab4cdef68 100644 --- a/scripts/un/energy/manifest.json +++ b/scripts/un/energy/manifest.json @@ -16,10 +16,11 @@ "import_inputs": [ { "template_mcf": "output_data/un_energy_output.tmcf", - "cleaned_csv": "output_data/un_energy_output.csv" + "cleaned_csv": "output_data/un_energy_output.csv", + "node_mcf": "output_data/*.mcf" } ], "cron_schedule": "0 4 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_bjs/nps/manifest.json b/scripts/us_bjs/nps/manifest.json index e3256beec4..6c1a5ed80a 100644 --- a/scripts/us_bjs/nps/manifest.json +++ b/scripts/us_bjs/nps/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "national_prison_stats.tmcf", - "cleaned_csv": "national_prison_stats.csv" + "cleaned_csv": "national_prison_stats.csv", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 9 5,25 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_bls/cpi/manifest.json b/scripts/us_bls/cpi/manifest.json index c3e6780e98..cbac7f152d 100644 --- a/scripts/us_bls/cpi/manifest.json +++ b/scripts/us_bls/cpi/manifest.json @@ -13,15 +13,18 @@ "import_inputs": [ { "template_mcf": "cpi_u.tmcf", - "cleaned_csv": "output/cpi_u.csv" + "cleaned_csv": "output/cpi_u.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "cpi_w.tmcf", - "cleaned_csv": "output/cpi_w.csv" + "cleaned_csv": "output/cpi_w.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "c_cpi_u.tmcf", - "cleaned_csv": "output/c_cpi_u.csv" + "cleaned_csv": "output/c_cpi_u.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -30,4 +33,4 @@ "cron_schedule": "0 3 2,16 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_bts/latch/manifest.json b/scripts/us_bts/latch/manifest.json index 0406c9e4fe..ef008fcdc0 100644 --- a/scripts/us_bts/latch/manifest.json +++ b/scripts/us_bts/latch/manifest.json @@ -15,19 +15,23 @@ "import_inputs": [ { "template_mcf": "output_files/us_transportation_household_part1.tmcf", - "cleaned_csv": "output_files/us_transportation_household_part1.csv" + "cleaned_csv": "output_files/us_transportation_household_part1.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/us_transportation_household_part2.tmcf", - "cleaned_csv": "output_files/us_transportation_household_part2.csv" + "cleaned_csv": "output_files/us_transportation_household_part2.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/us_transportation_household_part3.tmcf", - "cleaned_csv": "output_files/us_transportation_household_part3.csv" + "cleaned_csv": "output_files/us_transportation_household_part3.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/us_transportation_household_part4.tmcf", - "cleaned_csv": "output_files/us_transportation_household_part4.csv" + "cleaned_csv": "output_files/us_transportation_household_part4.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ diff --git a/scripts/us_cdc/500_places/manifest.json b/scripts/us_cdc/500_places/manifest.json index b848347f79..170ea014c1 100644 --- a/scripts/us_cdc/500_places/manifest.json +++ b/scripts/us_cdc/500_places/manifest.json @@ -17,19 +17,23 @@ "import_inputs": [ { "template_mcf": "cdc_places.tmcf", - "cleaned_csv": "gcs_output/cleaned_csv/County.csv" + "cleaned_csv": "gcs_output/cleaned_csv/County.csv", + "node_mcf": "gcs_output/cleaned_csv/*.mcf" }, { "template_mcf": "cdc_places_city.tmcf", - "cleaned_csv": "gcs_output/cleaned_csv/City.csv" + "cleaned_csv": "gcs_output/cleaned_csv/City.csv", + "node_mcf": "gcs_output/cleaned_csv/*.mcf" }, { "template_mcf": "cdc_places_zipcode.tmcf", - "cleaned_csv": "gcs_output/cleaned_csv/ZipCode.csv" + "cleaned_csv": "gcs_output/cleaned_csv/ZipCode.csv", + "node_mcf": "gcs_output/cleaned_csv/*.mcf" }, { "template_mcf": "cdc_places_censustract.tmcf", - "cleaned_csv": "gcs_output/cleaned_csv/CensusTract.csv" + "cleaned_csv": "gcs_output/cleaned_csv/CensusTract.csv", + "node_mcf": "gcs_output/cleaned_csv/*.mcf" } ], "cron_schedule": "0 4 * * 5", diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json index eae459b1c6..d5c9eb305e 100644 --- a/scripts/us_cdc/cdc500_state/manifest.json +++ b/scripts/us_cdc/cdc500_state/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "cdc500_state.tmcf", - "cleaned_csv": "CDC500State_Output/CDC500State_Output.csv" + "cleaned_csv": "CDC500State_Output/CDC500State_Output.csv", + "node_mcf": "CDC500State_Output/*.mcf" } ], "resource_limits": { @@ -27,4 +28,4 @@ "cron_schedule": "0 1 * * 1" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/cdc_standardized_precipitation/evapotranspiration_index/manifest.json b/scripts/us_cdc/cdc_standardized_precipitation/evapotranspiration_index/manifest.json index 65173c9781..57a6cf42b0 100644 --- a/scripts/us_cdc/cdc_standardized_precipitation/evapotranspiration_index/manifest.json +++ b/scripts/us_cdc/cdc_standardized_precipitation/evapotranspiration_index/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "StandardizedPrecipitationEvapotranspirationIndex.tmcf", - "cleaned_csv": "output/CDC_StandardizedPrecipitationEvapotranspirationIndex_output.csv" + "cleaned_csv": "output/CDC_StandardizedPrecipitationEvapotranspirationIndex_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -24,4 +25,4 @@ "cron_schedule": "0 10 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/cdc_standardized_precipitation/index/manifest.json b/scripts/us_cdc/cdc_standardized_precipitation/index/manifest.json index e84bf3828b..619f7e2005 100644 --- a/scripts/us_cdc/cdc_standardized_precipitation/index/manifest.json +++ b/scripts/us_cdc/cdc_standardized_precipitation/index/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "StandardizedPrecipitationIndex.tmcf", - "cleaned_csv": "output/CDC_StandardizedPrecipitationIndex_output.csv" + "cleaned_csv": "output/CDC_StandardizedPrecipitationIndex_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -24,4 +25,4 @@ "cron_schedule": "0 11 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/cdc_standardized_precipitation/palmer_drought/manifest.json b/scripts/us_cdc/cdc_standardized_precipitation/palmer_drought/manifest.json index 7446fc8e4a..14b46ec286 100644 --- a/scripts/us_cdc/cdc_standardized_precipitation/palmer_drought/manifest.json +++ b/scripts/us_cdc/cdc_standardized_precipitation/palmer_drought/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "PalmerDroughtSeverityIndex.tmcf", - "cleaned_csv": "output/CDC_PalmerDroughtSeverityIndex_output.csv" + "cleaned_csv": "output/CDC_PalmerDroughtSeverityIndex_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 12 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/environmental_health_toxicology/manifest.json b/scripts/us_cdc/environmental_health_toxicology/manifest.json index db6068ca41..939a9806a6 100644 --- a/scripts/us_cdc/environmental_health_toxicology/manifest.json +++ b/scripts/us_cdc/environmental_health_toxicology/manifest.json @@ -22,19 +22,23 @@ "import_inputs": [ { "template_mcf": "PM25CensusTractPollution_part1.tmcf", - "cleaned_csv": "output/PM2.5CensusTract_0.csv" + "cleaned_csv": "output/PM2.5CensusTract_0.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "PM25CensusTractPollution_part2.tmcf", - "cleaned_csv": "output/PM2.5CensusTract_1.csv" + "cleaned_csv": "output/PM2.5CensusTract_1.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "PM25CensusTractPollution_part3.tmcf", - "cleaned_csv": "output/PM2.5CensusTract_2.csv" + "cleaned_csv": "output/PM2.5CensusTract_2.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "PM25CensusTractPollution_part4.tmcf", - "cleaned_csv": "output/PM2.5CensusTract_3.csv" + "cleaned_csv": "output/PM2.5CensusTract_3.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 1 2 * *" @@ -61,19 +65,23 @@ "import_inputs": [ { "template_mcf": "OzoneCensusTractPollution_part1.tmcf", - "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_0.csv" + "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_0.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "OzoneCensusTractPollution_part2.tmcf", - "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_1.csv" + "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_1.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "OzoneCensusTractPollution_part3.tmcf", - "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_2.csv" + "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_2.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "OzoneCensusTractPollution_part4.tmcf", - "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_3.csv" + "cleaned_csv": "output/Census_Tract_Level_Ozone_Concentrations_3.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 1 3 * *" @@ -95,7 +103,8 @@ "import_inputs": [ { "template_mcf": "PM25CountyPollution.tmcf", - "cleaned_csv": "output/PM25county.csv" + "cleaned_csv": "output/PM25county.csv", + "node_mcf": "output/*.mcf" } ], "resource_limits": { @@ -122,7 +131,8 @@ "import_inputs": [ { "template_mcf": "OzoneCountyPollution.tmcf", - "cleaned_csv": "output/OzoneCounty.csv" + "cleaned_csv": "output/OzoneCounty.csv", + "node_mcf": "output/*.mcf" } ], "resource_limits": { @@ -133,4 +143,4 @@ "cron_schedule": "0 1 5 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/heat_related_illness/manifest.json b/scripts/us_cdc/heat_related_illness/manifest.json index cdeced0cd8..88da1924db 100644 --- a/scripts/us_cdc/heat_related_illness/manifest.json +++ b/scripts/us_cdc/heat_related_illness/manifest.json @@ -17,14 +17,16 @@ "import_inputs": [ { "template_mcf": "output_files/output_part1.tmcf", - "cleaned_csv": "output_files/cleaned.csv" + "cleaned_csv": "output_files/cleaned.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/output_part2.tmcf", - "cleaned_csv": "output_files/country_output.csv" + "cleaned_csv": "output_files/country_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 3 3 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_cdc/non_infectious_diseases/manifest.json b/scripts/us_cdc/non_infectious_diseases/manifest.json index a377ce8265..f0d0522183 100644 --- a/scripts/us_cdc/non_infectious_diseases/manifest.json +++ b/scripts/us_cdc/non_infectious_diseases/manifest.json @@ -16,10 +16,11 @@ "import_inputs": [ { "template_mcf": "data/output/NORS_NonInfectious_Disease.tmcf", - "cleaned_csv": "data/output/NORS_NonInfectious_Disease.csv" + "cleaned_csv": "data/output/NORS_NonInfectious_Disease.csv", + "node_mcf": "data/output/*.mcf" } ], "cron_schedule": "0 2 15 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/gender_income_inequality/manifest.json b/scripts/us_census/gender_income_inequality/manifest.json index faa6ef6410..b661ef5af7 100644 --- a/scripts/us_census/gender_income_inequality/manifest.json +++ b/scripts/us_census/gender_income_inequality/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "gender_income_inequality.tmcf", - "cleaned_csv": "gender_income_inequality.csv" + "cleaned_csv": "gender_income_inequality.csv", + "node_mcf": "*.mcf" } ], "source_files": [ diff --git a/scripts/us_census/pep/annual_population/manifest.json b/scripts/us_census/pep/annual_population/manifest.json index 3f1cef3508..96bcc8daa9 100644 --- a/scripts/us_census/pep/annual_population/manifest.json +++ b/scripts/us_census/pep/annual_population/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output_files/usa_annual_population.tmcf", - "cleaned_csv": "output_files/usa_annual_population.csv" + "cleaned_csv": "output_files/usa_annual_population.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 10 * * 2" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/monthly_population_estimate/manifest.json b/scripts/us_census/pep/monthly_population_estimate/manifest.json index 9af9b870fc..b8718f276d 100644 --- a/scripts/us_census/pep/monthly_population_estimate/manifest.json +++ b/scripts/us_census/pep/monthly_population_estimate/manifest.json @@ -16,10 +16,11 @@ "import_inputs": [ { "template_mcf": "output/USA_Population_Count.tmcf", - "cleaned_csv": "output/USA_Population_Count.csv" + "cleaned_csv": "output/USA_Population_Count.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 7 * * 6" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/pep_by_srh/manifest.json b/scripts/us_census/pep/pep_by_srh/manifest.json index 0275895f79..dbe30dada7 100644 --- a/scripts/us_census/pep/pep_by_srh/manifest.json +++ b/scripts/us_census/pep/pep_by_srh/manifest.json @@ -13,11 +13,13 @@ "import_inputs": [ { "template_mcf": "output_files/population_estimate_by_srh.tmcf", - "cleaned_csv": "output_files/population_estimate_by_srh.csv" + "cleaned_csv": "output_files/population_estimate_by_srh.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/population_estimate_by_srh_agg.tmcf", - "cleaned_csv": "output_files/population_estimate_by_srh_agg.csv" + "cleaned_csv": "output_files/population_estimate_by_srh_agg.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ @@ -26,4 +28,4 @@ "cron_schedule": "0 06 * * 1" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/population_estimate_by_race/manifest.json b/scripts/us_census/pep/population_estimate_by_race/manifest.json index 3792d57c28..92135b0a2c 100644 --- a/scripts/us_census/pep/population_estimate_by_race/manifest.json +++ b/scripts/us_census/pep/population_estimate_by_race/manifest.json @@ -16,18 +16,21 @@ "import_inputs": [ { "template_mcf": "output/USA_Population_Count_by_Race_National_state_2000.tmcf", - "cleaned_csv": "output/USA_Population_Count_by_Race_National_state_2000.csv" + "cleaned_csv": "output/USA_Population_Count_by_Race_National_state_2000.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/USA_Population_Count_by_Race_before_2000.tmcf", - "cleaned_csv": "output/USA_Population_Count_by_Race_before_2000.csv" + "cleaned_csv": "output/USA_Population_Count_by_Race_before_2000.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/USA_Population_Count_by_Race_county_after_2000.tmcf", - "cleaned_csv": "output/USA_Population_Count_by_Race_county_after_2000.csv" + "cleaned_csv": "output/USA_Population_Count_by_Race_county_after_2000.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 04 * * 1" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/population_estimates_by_asr/manifest.json b/scripts/us_census/pep/population_estimates_by_asr/manifest.json index 65933949af..3e2b0a87a9 100644 --- a/scripts/us_census/pep/population_estimates_by_asr/manifest.json +++ b/scripts/us_census/pep/population_estimates_by_asr/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output/usa_population_asr.tmcf", - "cleaned_csv": "output/usa_population_asr.csv" + "cleaned_csv": "output/usa_population_asr.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -27,4 +28,4 @@ } } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/us_pep_sex/manifest.json b/scripts/us_census/pep/us_pep_sex/manifest.json index 4744566d3f..4e915c7ce9 100644 --- a/scripts/us_census/pep/us_pep_sex/manifest.json +++ b/scripts/us_census/pep/us_pep_sex/manifest.json @@ -16,10 +16,11 @@ "import_inputs": [ { "template_mcf": "output/population_estimate_sex.tmcf", - "cleaned_csv": "output/population_estimate_sex.csv" + "cleaned_csv": "output/population_estimate_sex.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 03 * * 1" } ] -} \ No newline at end of file +} diff --git a/scripts/us_census/pep/us_pep_sexrace/manifest.json b/scripts/us_census/pep/us_pep_sexrace/manifest.json index 52be2bb440..d50c5b7c1a 100644 --- a/scripts/us_census/pep/us_pep_sexrace/manifest.json +++ b/scripts/us_census/pep/us_pep_sexrace/manifest.json @@ -13,19 +13,23 @@ "import_inputs": [ { "template_mcf": "output_files/final/national_after_2000.tmcf", - "cleaned_csv": "output_files/final/national_after_2000.csv" + "cleaned_csv": "output_files/final/national_after_2000.csv", + "node_mcf": "output_files/final/*.mcf" }, { "template_mcf": "output_files/final/national_before_2000.tmcf", - "cleaned_csv": "output_files/final/national_before_2000.csv" + "cleaned_csv": "output_files/final/national_before_2000.csv", + "node_mcf": "output_files/final/*.mcf" }, { "template_mcf": "output_files/final/state_county_after_2000.tmcf", - "cleaned_csv": "output_files/final/state_county_after_2000.csv" + "cleaned_csv": "output_files/final/state_county_after_2000.csv", + "node_mcf": "output_files/final/*.mcf" }, { "template_mcf": "output_files/final/state_county_before_2000.tmcf", - "cleaned_csv": "output_files/final/state_county_before_2000.csv" + "cleaned_csv": "output_files/final/state_county_before_2000.csv", + "node_mcf": "output_files/final/*.mcf" } ], "source_files": [ @@ -34,4 +38,4 @@ "cron_schedule": "0 10 * * 1" } ] -} \ No newline at end of file +} diff --git a/scripts/us_eia/opendata/manifest.json b/scripts/us_eia/opendata/manifest.json index 5dd9555830..50b2bab1ff 100644 --- a/scripts/us_eia/opendata/manifest.json +++ b/scripts/us_eia/opendata/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/COAL/COAL.tmcf", - "cleaned_csv": "gcs_output/COAL/COAL.csv" + "cleaned_csv": "gcs_output/COAL/COAL.csv", + "node_mcf": "gcs_output/COAL/*.mcf" } ], "cron_schedule": "0 14 5,20 * *" @@ -37,7 +38,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/ELEC/ELEC.tmcf", - "cleaned_csv": "gcs_output/ELEC/ELEC.csv" + "cleaned_csv": "gcs_output/ELEC/ELEC.csv", + "node_mcf": "gcs_output/ELEC/*.mcf" } ], "resource_limits": { @@ -63,7 +65,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/NG/NG.tmcf", - "cleaned_csv": "gcs_output/NG/NG.csv" + "cleaned_csv": "gcs_output/NG/NG.csv", + "node_mcf": "gcs_output/NG/*.mcf" } ], "cron_schedule": "0 8 5,20 * *" @@ -84,7 +87,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/NUC_STATUS/NUC_STATUS.tmcf", - "cleaned_csv": "gcs_output/NUC_STATUS/NUC_STATUS.csv" + "cleaned_csv": "gcs_output/NUC_STATUS/NUC_STATUS.csv", + "node_mcf": "gcs_output/NUC_STATUS/*.mcf" } ], "cron_schedule": "0 9 5,20 * *" @@ -105,7 +109,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/PET/PET.tmcf", - "cleaned_csv": "gcs_output/PET/PET.csv" + "cleaned_csv": "gcs_output/PET/PET.csv", + "node_mcf": "gcs_output/PET/*.mcf" } ], "cron_schedule": "0 10 5,20 * *" @@ -126,7 +131,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/INTL/INTL.tmcf", - "cleaned_csv": "gcs_output/INTL/INTL.csv" + "cleaned_csv": "gcs_output/INTL/INTL.csv", + "node_mcf": "gcs_output/INTL/*.mcf" } ], "cron_schedule": "0 11 5,20 * * *" @@ -147,7 +153,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/SEDS/SEDS.tmcf", - "cleaned_csv": "gcs_output/SEDS/SEDS.csv" + "cleaned_csv": "gcs_output/SEDS/SEDS.csv", + "node_mcf": "gcs_output/SEDS/*.mcf" } ], "cron_schedule": "0 12 5,20 * *" @@ -168,10 +175,11 @@ "import_inputs": [ { "template_mcf": "gcs_output/TOTAL/TOTAL.tmcf", - "cleaned_csv": "gcs_output/TOTAL/TOTAL.csv" + "cleaned_csv": "gcs_output/TOTAL/TOTAL.csv", + "node_mcf": "gcs_output/TOTAL/*.mcf" } ], "cron_schedule": "0 13 5,20 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/air_emissions_inventory/national_state/manifest.json b/scripts/us_epa/air_emissions_inventory/national_state/manifest.json index 110e3a3785..261074ecf2 100644 --- a/scripts/us_epa/air_emissions_inventory/national_state/manifest.json +++ b/scripts/us_epa/air_emissions_inventory/national_state/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/airpollution_emission_trends_tier1.tmcf", - "cleaned_csv": "output/airpollution_emission_trends_tier1.csv" + "cleaned_csv": "output/airpollution_emission_trends_tier1.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -23,4 +24,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/airdata/manifest.json b/scripts/us_epa/airdata/manifest.json index 84260a0991..445a083bd1 100644 --- a/scripts/us_epa/airdata/manifest.json +++ b/scripts/us_epa/airdata/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "EPA_AirQuality.tmcf", - "cleaned_csv": "EPA_AirQuality.csv" + "cleaned_csv": "EPA_AirQuality.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "15 22 10 1,7 *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/airqualityindex/manifest.json b/scripts/us_epa/airqualityindex/manifest.json index dbf52bba55..5e959764fd 100644 --- a/scripts/us_epa/airqualityindex/manifest.json +++ b/scripts/us_epa/airqualityindex/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "EPA_AQI.tmcf", - "cleaned_csv": "output/EPA_AQI.csv" + "cleaned_csv": "output/EPA_AQI.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "15 22 10 1,7 *", @@ -27,4 +28,4 @@ } } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/ejscreen/manifest.json b/scripts/us_epa/ejscreen/manifest.json index 1e543a0516..7a41f0db9a 100644 --- a/scripts/us_epa/ejscreen/manifest.json +++ b/scripts/us_epa/ejscreen/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output_files/ejscreen.tmcf", - "cleaned_csv": "output_files/ejscreen_airpollutants.csv" + "cleaned_csv": "output_files/ejscreen_airpollutants.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -27,4 +28,4 @@ } } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/ghgrp/manifest.json b/scripts/us_epa/ghgrp/manifest.json index 9ff71fdc68..da9d4fc478 100644 --- a/scripts/us_epa/ghgrp/manifest.json +++ b/scripts/us_epa/ghgrp/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "import_data/observations.tmcf", - "cleaned_csv": "import_data/all_data.csv" + "cleaned_csv": "import_data/all_data.csv", + "node_mcf": "import_data/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -25,4 +26,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/scripts/us_epa/national_emissions_inventory/manifest.json b/scripts/us_epa/national_emissions_inventory/manifest.json index aa997fecf3..6dd6c2938d 100644 --- a/scripts/us_epa/national_emissions_inventory/manifest.json +++ b/scripts/us_epa/national_emissions_inventory/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "gcs_output/output_files/national_emissions.tmcf", - "cleaned_csv": "gcs_output/output_files/national_emissions.csv" + "cleaned_csv": "gcs_output/output_files/national_emissions.csv", + "node_mcf": "gcs_output/output_files/*.mcf" } ], "cron_schedule": "0 0 1 1-12/3 *", @@ -29,5 +30,3 @@ } ] } - - diff --git a/scripts/us_fema/national_risk_index/manifest.json b/scripts/us_fema/national_risk_index/manifest.json index 0d43ec106b..3d0545d230 100644 --- a/scripts/us_fema/national_risk_index/manifest.json +++ b/scripts/us_fema/national_risk_index/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/fema_nri_counties.tmcf", - "cleaned_csv": "output/*.csv" + "cleaned_csv": "output/*.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 4 2,16 * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_gs/earthquake/manifest.json b/scripts/us_gs/earthquake/manifest.json index c18a8040a2..78d74a990d 100644 --- a/scripts/us_gs/earthquake/manifest.json +++ b/scripts/us_gs/earthquake/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "earthquake.tmcf", - "cleaned_csv": "earthquake.csv" + "cleaned_csv": "earthquake.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "0 */3 * * *" } ] -} \ No newline at end of file +} diff --git a/scripts/us_hud/income/manifest.json b/scripts/us_hud/income/manifest.json index a571c53088..d3b14a1c7d 100644 --- a/scripts/us_hud/income/manifest.json +++ b/scripts/us_hud/income/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "hud.tmcf", - "cleaned_csv": "output/output_all_years.csv" + "cleaned_csv": "output/output_all_years.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -22,4 +23,4 @@ "cron_schedule": "15 23 * * 7" } ] -} \ No newline at end of file +} diff --git a/scripts/us_nces/demographics/private_school/manifest.json b/scripts/us_nces/demographics/private_school/manifest.json index 195b29fb2e..e45b0b4a31 100644 --- a/scripts/us_nces/demographics/private_school/manifest.json +++ b/scripts/us_nces/demographics/private_school/manifest.json @@ -17,11 +17,13 @@ "import_inputs": [ { "template_mcf": "gcs_folder/output_files/us_nces_demographics_private_school.tmcf", - "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_private_school.csv" + "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_private_school.csv", + "node_mcf": "gcs_folder/output_files/*.mcf" }, { "template_mcf": "gcs_folder/output_place/us_nces_demographics_private_place.tmcf", - "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_private_place.csv" + "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_private_place.csv", + "node_mcf": "gcs_folder/output_place/*.mcf" } ], "cron_schedule": "30 3 1 3,6,9,12 *", @@ -32,4 +34,4 @@ } } ] -} +} diff --git a/scripts/us_nces/demographics/public_school/manifest.json b/scripts/us_nces/demographics/public_school/manifest.json index 33b3eba897..dc241218f9 100644 --- a/scripts/us_nces/demographics/public_school/manifest.json +++ b/scripts/us_nces/demographics/public_school/manifest.json @@ -17,11 +17,13 @@ "import_inputs": [ { "template_mcf": "gcs_folder/output_files/us_nces_demographics_public_school.tmcf", - "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_public_school.csv" + "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_public_school.csv", + "node_mcf": "gcs_folder/output_files/*.mcf" }, { "template_mcf": "gcs_folder/output_place/us_nces_demographics_public_place.tmcf", - "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_public_place.csv" + "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_public_place.csv", + "node_mcf": "gcs_folder/output_place/*.mcf" } ], "cron_schedule": "30 3 2 3,6,9,12 *", diff --git a/scripts/us_nces/demographics/school_district/manifest.json b/scripts/us_nces/demographics/school_district/manifest.json index d42acaee49..acece15f6a 100644 --- a/scripts/us_nces/demographics/school_district/manifest.json +++ b/scripts/us_nces/demographics/school_district/manifest.json @@ -17,11 +17,13 @@ "import_inputs": [ { "template_mcf": "gcs_folder/output_files/us_nces_demographics_district_school.tmcf", - "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_district_school.csv" + "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_district_school.csv", + "node_mcf": "gcs_folder/output_files/*.mcf" }, { "template_mcf": "gcs_folder/output_place/us_nces_demographics_district_place.tmcf", - "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_district_place.csv" + "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_district_place.csv", + "node_mcf": "gcs_folder/output_place/*.mcf" } ], "cron_schedule": "30 3 3 3,6,9,12 *", diff --git a/scripts/us_usda/nass_census/manifest.json b/scripts/us_usda/nass_census/manifest.json index c7a3c4c0d2..87489ada59 100644 --- a/scripts/us_usda/nass_census/manifest.json +++ b/scripts/us_usda/nass_census/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "agriculture.tmcf", - "cleaned_csv": "output/agriculture.csv" + "cleaned_csv": "output/agriculture.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -22,4 +23,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/scripts/us_usda/quickstats/manifest.json b/scripts/us_usda/quickstats/manifest.json index f7adcb75ba..dd935cebfe 100644 --- a/scripts/us_usda/quickstats/manifest.json +++ b/scripts/us_usda/quickstats/manifest.json @@ -16,10 +16,11 @@ "import_inputs": [ { "template_mcf": "ag.tmcf", - "cleaned_csv": "output/*.csv" + "cleaned_csv": "output/*.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "30 3 10 3,6,9,12 *" } ] -} \ No newline at end of file +} diff --git a/scripts/world_bank/datasets/manifest.json b/scripts/world_bank/datasets/manifest.json index 74776f0d0d..95c3ebbf21 100644 --- a/scripts/world_bank/datasets/manifest.json +++ b/scripts/world_bank/datasets/manifest.json @@ -18,11 +18,13 @@ "import_inputs": [ { "template_mcf": "wb_part1.tmcf", - "cleaned_csv": "gcs_output/output/transformed_data_for_all_final.csv" + "cleaned_csv": "gcs_output/output/transformed_data_for_all_final.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "wb_part2.tmcf", - "cleaned_csv": "gcs_output/output/bq-results-20250423.csv" + "cleaned_csv": "gcs_output/output/bq-results-20250423.csv", + "node_mcf": "gcs_output/output/*.mcf" } ], "cron_schedule": "0 9 5,25 * *", @@ -33,4 +35,4 @@ } } ] -} \ No newline at end of file +} diff --git a/scripts/world_bank/wdi/download_indicators/manifest.json b/scripts/world_bank/wdi/download_indicators/manifest.json index 4168aa4c54..c7af06e6a5 100644 --- a/scripts/world_bank/wdi/download_indicators/manifest.json +++ b/scripts/world_bank/wdi/download_indicators/manifest.json @@ -12,7 +12,8 @@ ], "import_inputs": [ { - "cleaned_csv": "cleaned_wdi.csv" + "cleaned_csv": "cleaned_wdi.csv", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 0 1 * *" diff --git a/scripts/world_bank/wdi/manifest.json b/scripts/world_bank/wdi/manifest.json index bc3927141e..5a2bd9f04c 100644 --- a/scripts/world_bank/wdi/manifest.json +++ b/scripts/world_bank/wdi/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output/WorldBank.tmcf", - "cleaned_csv": "output/WorldBank.csv" + "cleaned_csv": "output/WorldBank.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 11 * * 2" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/PulmonaryTuberculosis_Bacteriologically_Confirmed/manifest.json b/statvar_imports/PulmonaryTuberculosis_Bacteriologically_Confirmed/manifest.json index 532ab72a8e..9cada57bf1 100644 --- a/statvar_imports/PulmonaryTuberculosis_Bacteriologically_Confirmed/manifest.json +++ b/statvar_imports/PulmonaryTuberculosis_Bacteriologically_Confirmed/manifest.json @@ -17,19 +17,24 @@ "import_inputs": [ { "template_mcf": "TB_Bacteriologically_Confirmed_output.tmcf", - "cleaned_csv": "TB_Bacteriologically_Confirmed_output.csv" + "cleaned_csv": "TB_Bacteriologically_Confirmed_output.csv", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 0 1 1,4,7,10 *", - "resource_limits": {"cpu": 4, "memory": 8, "disk":100} + "resource_limits": { + "cpu": 4, + "memory": 8, + "disk": 100 + } } ], "config_override": { - "invoke_import_validation": true, - "invoke_import_tool": true, - "invoke_differ_tool": true, - "skip_input_upload": false, - "skip_gcs_upload": false, - "cleanup_gcs_volume_mount": false + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false } } diff --git a/statvar_imports/bis/bis_central_bank_policy_rate/manifest.json b/statvar_imports/bis/bis_central_bank_policy_rate/manifest.json index f2447aa71e..d74e7e56e3 100644 --- a/statvar_imports/bis/bis_central_bank_policy_rate/manifest.json +++ b/statvar_imports/bis/bis_central_bank_policy_rate/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "output/central_bank_policy_rate.tmcf", - "cleaned_csv": "output/central_bank_policy_rate.csv" + "cleaned_csv": "output/central_bank_policy_rate.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 05 * * 6", diff --git a/statvar_imports/brazil_sidra_ibge/manifest.json b/statvar_imports/brazil_sidra_ibge/manifest.json index a18fddf2ed..d6a6f72cd3 100644 --- a/statvar_imports/brazil_sidra_ibge/manifest.json +++ b/statvar_imports/brazil_sidra_ibge/manifest.json @@ -17,19 +17,23 @@ "import_inputs": [ { "template_mcf": "output/average_real_income_output.tmcf", - "cleaned_csv": "output/average_real_income_output.csv" + "cleaned_csv": "output/average_real_income_output.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/mass_income_output.tmcf", - "cleaned_csv": "output/mass_income_output.csv" + "cleaned_csv": "output/mass_income_output.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/population_economic_sector_output.tmcf", - "cleaned_csv": "output/population_economic_sector_output.csv" + "cleaned_csv": "output/population_economic_sector_output.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/employment_and_unemployment_labor_force_output.tmcf", - "cleaned_csv": "output/employment_and_unemployment_labor_force_output.csv" + "cleaned_csv": "output/employment_and_unemployment_labor_force_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -41,4 +45,4 @@ "cron_schedule": "30 09 25 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/brazil_visdata/brazil_rural_development_program/manifest.json b/statvar_imports/brazil_visdata/brazil_rural_development_program/manifest.json index 8ef3c60c59..0f453e20e2 100644 --- a/statvar_imports/brazil_visdata/brazil_rural_development_program/manifest.json +++ b/statvar_imports/brazil_visdata/brazil_rural_development_program/manifest.json @@ -1,81 +1,96 @@ { - "import_specifications": [ - { - "import_name": "Brazil_RuralDevelopmentProgram", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://aplicacoes.cidadania.gov.br/vis/data3/data-explorer.php", - "provenance_description": "Different population benefiting from Brazil Rural Development Program.", - "scripts": [ - "run.sh" - ], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_Brazil_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_Brazil_output.csv" - }, - { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_Municipality_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_Municipality_output.csv" - }, - { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_State_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_State_output.csv" - }, - { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Brazil_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Brazil_output.csv" - }, - { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Municipality_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Municipality_output.csv" - }, - { - "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_State_output.tmcf", - "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_State_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_yearly_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_yearly_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_Municipality_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_Municipality_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_State_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_State_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_State_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_State_output.csv" - }, - { - "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_yearly_Municipality_output.tmcf", - "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_yearly_Municipality_output.csv" - }, - { - "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_Brazil_output.tmcf", - "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_Brazil_output.csv" - }, - { - "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_Municipality_output.tmcf", - "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_Municipality_output.csv" - }, - { - "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_State_output.tmcf", - "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_State_output.csv" + "import_name": "Brazil_RuralDevelopmentProgram", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://aplicacoes.cidadania.gov.br/vis/data3/data-explorer.php", + "provenance_description": "Different population benefiting from Brazil Rural Development Program.", + "scripts": [ + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_Brazil_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_Brazil_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_Municipality_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_Municipality_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_Gender_State_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_Gender_State_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Brazil_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Brazil_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Municipality_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_Municipality_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_State_output.tmcf", + "cleaned_csv": "output_files/Families_RuralDevelopmentProgram_SpecificPopulation_State_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_yearly_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_brazil_yearly_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_Municipality_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_Municipality_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_State_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_latest_State_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_State_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_State_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_yearly_Municipality_output.tmcf", + "cleaned_csv": "output_files/FinancialResources_Beneficiary_RuralDevelopmentProgram_yearly_Municipality_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_Brazil_output.tmcf", + "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_Brazil_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_Municipality_output.tmcf", + "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_Municipality_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/TotalFamilies_Rural_Development_Program_State_output.tmcf", + "cleaned_csv": "output_files/TotalFamilies_Rural_Development_Program_State_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 06 * * 5" } - ], - "cron_schedule": "0 06 * * 5" - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/brfss_nchs_asthma_prevalence/nchs_brfss_asthma/manifest.json b/statvar_imports/brfss_nchs_asthma_prevalence/nchs_brfss_asthma/manifest.json index ff5c04037d..eaed8e9098 100644 --- a/statvar_imports/brfss_nchs_asthma_prevalence/nchs_brfss_asthma/manifest.json +++ b/statvar_imports/brfss_nchs_asthma_prevalence/nchs_brfss_asthma/manifest.json @@ -1,42 +1,52 @@ { - "import_specifications": [ - { - "import_name": "NCHS_BRFSS_Asthma", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.cdc.gov/asthma/brfss/default.htm", - "provenance_description": "This dataset mainly focused on Adult and child Self-Reported Lifetime Asthma Prevalence.", - "scripts": ["download_script.py", "run.sh"], - "source_files": [ - "input_files/*/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/brfss_asthma_by_age_and_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_age_and_state_output.csv" - }, - { - "template_mcf": "output_files/brfss_asthma_by_education_and_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_education_and_state_output.csv" - }, - { - "template_mcf": "output_files/brfss_asthma_by_income_and_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_income_and_state_output.csv" - }, - { - "template_mcf": "output_files/brfss_asthma_by_race_and_ethnicity_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_race_and_ethnicity_state_output.csv" - }, - { - "template_mcf": "output_files/brfss_asthma_by_sex_and_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_sex_and_state_output.csv" - }, - { - "template_mcf": "output_files/brfss_asthma_by_state_output.tmcf", - "cleaned_csv": "output_files/brfss_asthma_by_state_output.csv" + "import_name": "NCHS_BRFSS_Asthma", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.cdc.gov/asthma/brfss/default.htm", + "provenance_description": "This dataset mainly focused on Adult and child Self-Reported Lifetime Asthma Prevalence.", + "scripts": [ + "download_script.py", + "run.sh" + ], + "source_files": [ + "input_files/*/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/brfss_asthma_by_age_and_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_age_and_state_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/brfss_asthma_by_education_and_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_education_and_state_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/brfss_asthma_by_income_and_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_income_and_state_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/brfss_asthma_by_race_and_ethnicity_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_race_and_ethnicity_state_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/brfss_asthma_by_sex_and_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_sex_and_state_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/brfss_asthma_by_state_output.tmcf", + "cleaned_csv": "output_files/brfss_asthma_by_state_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 02 * * 2" } - ], - "cron_schedule": "0 02 * * 2" - } - ] + ] } - diff --git a/statvar_imports/canada_statistics_age_gender/manifest.json b/statvar_imports/canada_statistics_age_gender/manifest.json index 565e42b59f..e3e90beabc 100644 --- a/statvar_imports/canada_statistics_age_gender/manifest.json +++ b/statvar_imports/canada_statistics_age_gender/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output/canada_annual_population_age_gender_output.tmcf", - "cleaned_csv": "output/canada_annual_population_age_gender_output.csv" + "cleaned_csv": "output/canada_annual_population_age_gender_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/ccd/enrollment/manifest.json b/statvar_imports/ccd/enrollment/manifest.json index c8cce2d201..1a5040e586 100644 --- a/statvar_imports/ccd/enrollment/manifest.json +++ b/statvar_imports/ccd/enrollment/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output/CCD_Enrollment_output.tmcf", - "cleaned_csv": "output/CCD_Enrollment_output.csv" + "cleaned_csv": "output/CCD_Enrollment_output.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 06 * * 5" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/cdc/social_vulnerability_index/manifest.json b/statvar_imports/cdc/social_vulnerability_index/manifest.json index 2d4354aef0..f649a20d69 100644 --- a/statvar_imports/cdc/social_vulnerability_index/manifest.json +++ b/statvar_imports/cdc/social_vulnerability_index/manifest.json @@ -15,11 +15,13 @@ "import_inputs": [ { "template_mcf": "output/SVI_2010_us_county.tmcf", - "cleaned_csv": "output/SVI_2010_us_county.csv" + "cleaned_csv": "output/SVI_2010_us_county.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/SVI_us_county.tmcf", - "cleaned_csv": "output/SVI_us_county.csv" + "cleaned_csv": "output/SVI_us_county.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/census_v2/sahie/manifest.json b/statvar_imports/census_v2/sahie/manifest.json index 28e686e687..ab6ea2e3c1 100644 --- a/statvar_imports/census_v2/sahie/manifest.json +++ b/statvar_imports/census_v2/sahie/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output_files/census_sahie.tmcf", - "cleaned_csv": "output_files/census_sahie.csv" + "cleaned_csv": "output_files/census_sahie.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ @@ -28,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/census_v2/saipe/manifest.json b/statvar_imports/census_v2/saipe/manifest.json index 668c108d65..a4e5d91242 100644 --- a/statvar_imports/census_v2/saipe/manifest.json +++ b/statvar_imports/census_v2/saipe/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output_files/census_saipe_output.tmcf", - "cleaned_csv": "output_files/census_saipe_output.csv" + "cleaned_csv": "output_files/census_saipe_output.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/child_birth/manifest.json b/statvar_imports/child_birth/manifest.json index ea34e030ee..33dffd589c 100644 --- a/statvar_imports/child_birth/manifest.json +++ b/statvar_imports/child_birth/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "output_files/child_birth.tmcf", - "cleaned_csv": "output_files/child_birth.csv" + "cleaned_csv": "output_files/child_birth.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 07 * * 2" diff --git a/statvar_imports/climate_trace/manifest.json b/statvar_imports/climate_trace/manifest.json index 0a847c95de..ba551b6916 100644 --- a/statvar_imports/climate_trace/manifest.json +++ b/statvar_imports/climate_trace/manifest.json @@ -1,27 +1,28 @@ { - "import_specifications": [ - { - "import_name": "ClimateTrace", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://climatetrace.org/", - "provenance_description": "Climate Trace data provides comprehensive, granular, and up-to-date greenhouse gas emissions data using satellite imagery and other public data sources.", - "scripts": [ - "download_and_segregate_by_gas.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/all*.csv --pv_map=climate_pvmap_sectors.csv --config_file=common_metadata.csv --output_path=output/output_sectors --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf", - "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/all*.csv --pv_map=climate_pvmap_subsectors.csv --config_file=common_metadata.csv --output_path=output/output_subsectors --statvar_dcid_remap_csv=statvar_remap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "common_output.tmcf", - "cleaned_csv": "output/*.csv" + "import_name": "ClimateTrace", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://climatetrace.org/", + "provenance_description": "Climate Trace data provides comprehensive, granular, and up-to-date greenhouse gas emissions data using satellite imagery and other public data sources.", + "scripts": [ + "download_and_segregate_by_gas.py", + "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/all*.csv --pv_map=climate_pvmap_sectors.csv --config_file=common_metadata.csv --output_path=output/output_sectors --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf", + "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/all*.csv --pv_map=climate_pvmap_subsectors.csv --config_file=common_metadata.csv --output_path=output/output_subsectors --statvar_dcid_remap_csv=statvar_remap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" + ], + "import_inputs": [ + { + "template_mcf": "common_output.tmcf", + "cleaned_csv": "output/*.csv", + "node_mcf": "output/*.mcf" + } + ], + "source_file": [ + "input_files/*" + ], + "cron_schedule": "30 05 7,24 * *" } - ], - "source_file": [ - "input_files/*" - ], - "cron_schedule": "30 05 7,24 * *" - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/commerce_eda/manifest.json b/statvar_imports/commerce_eda/manifest.json index df379739eb..1afa69743e 100644 --- a/statvar_imports/commerce_eda/manifest.json +++ b/statvar_imports/commerce_eda/manifest.json @@ -1,31 +1,39 @@ { - "import_specifications": [ - { - "import_name": "Commerce_EDA", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.census.gov/retail/", - "provenance_description": "Commerce - EDA has led the federal economic development agenda by promoting innovation and competitiveness, preparing American regions for growth and success in the worldwide economy.", - "scripts": ["process.py","../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/Investment1.csv --pv_map=Investmentpvmap.csv --config_file=Investmentmetadata.csv --output_path=output/Investment_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ", - "../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/EstimatedOutcome.csv --pv_map=Estimatedpvmap.csv --config_file=Estimatedmetadata.csv --output_path=output/Estimated_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ", - "../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/Poverty.csv --pv_map=Povertypvmap.csv --config_file=Povertymetadata.csv --output_path=output/Poverty_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf " ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/Investment_output.tmcf", - "cleaned_csv": "output/Investment_output.csv" - }, - { - "template_mcf": "output/Estimated_output.tmcf", - "cleaned_csv": "output/Estimated_output.csv" - }, - { - "template_mcf": "output/Poverty_output.tmcf", - "cleaned_csv": "output/Poverty_output.csv" + "import_name": "Commerce_EDA", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.census.gov/retail/", + "provenance_description": "Commerce - EDA has led the federal economic development agenda by promoting innovation and competitiveness, preparing American regions for growth and success in the worldwide economy.", + "scripts": [ + "process.py", + "../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/Investment1.csv --pv_map=Investmentpvmap.csv --config_file=Investmentmetadata.csv --output_path=output/Investment_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ", + "../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/EstimatedOutcome.csv --pv_map=Estimatedpvmap.csv --config_file=Estimatedmetadata.csv --output_path=output/Estimated_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ", + "../../tools/statvar_importer/stat_var_processor.py --input_data=gcs_output/Poverty.csv --pv_map=Povertypvmap.csv --config_file=Povertymetadata.csv --output_path=output/Poverty_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf " + ], + "import_inputs": [ + { + "template_mcf": "output/Investment_output.tmcf", + "cleaned_csv": "output/Investment_output.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/Estimated_output.tmcf", + "cleaned_csv": "output/Estimated_output.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/Poverty_output.tmcf", + "cleaned_csv": "output/Poverty_output.csv", + "node_mcf": "output/*.mcf" + } + ], + "source_file": [ + "input_file/*" + ], + "cron_schedule": "30 05 * * 1" } - ], - "source_file":[ - "input_file/*" - ], - "cron_schedule": "30 05 * * 1" - } - ] + ] } diff --git a/statvar_imports/crdc/instructional_wifi_devices/manifest.json b/statvar_imports/crdc/instructional_wifi_devices/manifest.json index ed7187537b..7ed020cc7c 100644 --- a/statvar_imports/crdc/instructional_wifi_devices/manifest.json +++ b/statvar_imports/crdc/instructional_wifi_devices/manifest.json @@ -3,19 +3,21 @@ { "import_name": "crdc_instructional_wifi_devices", "curator_emails": [ - "support@datacommons.org" + "support@datacommons.org" ], "provenance_url": "https://civilrightsdata.ed.gov/data", "provenance_description": "The Civil Rights Data Collection (CRDC) has released data concerning the total number of devices (computers, laptops, tablets, etc.) that can connect via Wi-Fi and are used by students for instructional purposes", "scripts": [ - "download.py", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=common_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/instructional_wifi_devices" + "download.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=common_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/instructional_wifi_devices" ], "import_inputs": [ - { - "template_mcf": "output/instructional_wifi_devices.tmcf", - "cleaned_csv": "output/instructional_wifi_devices.csv" - } - ], + { + "template_mcf": "output/instructional_wifi_devices.tmcf", + "cleaned_csv": "output/instructional_wifi_devices.csv", + "node_mcf": "output/*.mcf" + } + ], "source_files": [ "input_files/*.csv" ], diff --git a/statvar_imports/crdc_import/crdc_harassment_or_bullying/manifest.json b/statvar_imports/crdc_import/crdc_harassment_or_bullying/manifest.json index 674662dbe3..7180d3da60 100644 --- a/statvar_imports/crdc_import/crdc_harassment_or_bullying/manifest.json +++ b/statvar_imports/crdc_import/crdc_harassment_or_bullying/manifest.json @@ -8,14 +8,17 @@ "provenance_url": "https://civilrightsdata.ed.gov/data", "provenance_description": "The Civil Rights Data Collection (CRDC) provides data on key education and civil rights issues in our nation's public schools. This import focuses on harassment or bullying data.", "scripts": [ - "download.py","preprocess.py","run.sh" - ], + "download.py", + "preprocess.py", + "run.sh" + ], "import_inputs": [ { "template_mcf": "processed_output/crdc_harassment_or_bullying_2022.tmcf", - "cleaned_csv": "processed_output/*.csv" + "cleaned_csv": "processed_output/*.csv", + "node_mcf": "processed_output/*.mcf" } - ], + ], "source_files": [ "input_files/*.csv" ], @@ -27,4 +30,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/crdc_import/manifest.json b/statvar_imports/crdc_import/manifest.json index 22de94684b..89d20be3ac 100644 --- a/statvar_imports/crdc_import/manifest.json +++ b/statvar_imports/crdc_import/manifest.json @@ -16,15 +16,18 @@ "import_inputs": [ { "template_mcf": "output_files/chronic_absenteeism_output.tmcf", - "cleaned_csv": "output_files/chronic_absenteeism_output.csv" + "cleaned_csv": "output_files/chronic_absenteeism_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/offenses_output.tmcf", - "cleaned_csv": "output_files/offenses_output.csv" + "cleaned_csv": "output_files/offenses_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/restraint_and_seclusion_output.tmcf", - "cleaned_csv": "output_files/restraint_and_seclusion_output.csv" + "cleaned_csv": "output_files/restraint_and_seclusion_output.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index efeca24ba2..1a5cf80545 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -20,78 +20,96 @@ "import_inputs": [ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", - "cleaned_csv": "output_files/agriculture/agriculture_output.csv" + "cleaned_csv": "output_files/agriculture/agriculture_output.csv", + "node_mcf": "output_files/agriculture/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv" + "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", - "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv" + "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", + "node_mcf": "output_files/price_and_wages/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" } ], "cron_schedule": "0 10 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/denmark_demographics/manifest.json b/statvar_imports/denmark_demographics/manifest.json index 2d992dcb37..c309f128ca 100644 --- a/statvar_imports/denmark_demographics/manifest.json +++ b/statvar_imports/denmark_demographics/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "output/population_sex_age_time_output.tmcf", - "cleaned_csv": "output/*_output.csv" + "cleaned_csv": "output/*_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/doctoratedegreeemployment/manifest.json b/statvar_imports/doctoratedegreeemployment/manifest.json index 7cb1f7f41a..7444b9bb37 100644 --- a/statvar_imports/doctoratedegreeemployment/manifest.json +++ b/statvar_imports/doctoratedegreeemployment/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/doctoratedegreeemployment.tmcf", - "cleaned_csv": "output/doctoratedegreeemployment.csv" + "cleaned_csv": "output/doctoratedegreeemployment.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 05 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/fao_currency_and_exchange_rate/fao_currency_statvar/manifest.json b/statvar_imports/fao_currency_and_exchange_rate/fao_currency_statvar/manifest.json index 01c7deb723..315182b73f 100644 --- a/statvar_imports/fao_currency_and_exchange_rate/fao_currency_statvar/manifest.json +++ b/statvar_imports/fao_currency_and_exchange_rate/fao_currency_statvar/manifest.json @@ -18,10 +18,11 @@ "import_inputs": [ { "template_mcf": "output_files/CurrencyFAO_output.tmcf", - "cleaned_csv": "output_files/CurrencyFAO_output.csv" + "cleaned_csv": "output_files/CurrencyFAO_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 07 * * 4" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/fbi/fbigovcrime/manifest.json b/statvar_imports/fbi/fbigovcrime/manifest.json index f1993186f2..cef01b833d 100644 --- a/statvar_imports/fbi/fbigovcrime/manifest.json +++ b/statvar_imports/fbi/fbigovcrime/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/fbi_gov_crime.tmcf", - "cleaned_csv": "output/fbi_gov_crime.csv" + "cleaned_csv": "output/fbi_gov_crime.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/fema/flood_insurance_claims/manifest.json b/statvar_imports/fema/flood_insurance_claims/manifest.json index 21107bf515..fe134301d5 100644 --- a/statvar_imports/fema/flood_insurance_claims/manifest.json +++ b/statvar_imports/fema/flood_insurance_claims/manifest.json @@ -1,25 +1,27 @@ { "import_specifications": [ - { - "import_name": "USFEMA_FloodInsuranceClaims", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.fema.gov/openfema-data-page/fima-nfip-redacted-claims-v2", - "provenance_description": "National flood insurance claims data", - "scripts": [ - "fema_download.py", - "process.py --input_data='input_file/fema_nfip_claims.csv' --config_file='us_flood_nfip_config.py' --pv_map='us_flood_nfip_pv_map_floodzone.py,ratedFloodZone:us_flood_nfip_floodzone_pv_map.py,observationAbout:us_state_codes.py' --output_path='output/nfip_output'" - ], - "import_inputs": [ - { - "template_mcf": "output/nfip_output.tmcf", - "cleaned_csv": "output/nfip_output.csv" - } - ], - "source_files": [ - "input_file/fema_nfip_claims.csv" - ], - "cron_schedule": "0 11 1 * *" - } + { + "import_name": "USFEMA_FloodInsuranceClaims", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.fema.gov/openfema-data-page/fima-nfip-redacted-claims-v2", + "provenance_description": "National flood insurance claims data", + "scripts": [ + "fema_download.py", + "process.py --input_data='input_file/fema_nfip_claims.csv' --config_file='us_flood_nfip_config.py' --pv_map='us_flood_nfip_pv_map_floodzone.py,ratedFloodZone:us_flood_nfip_floodzone_pv_map.py,observationAbout:us_state_codes.py' --output_path='output/nfip_output'" + ], + "import_inputs": [ + { + "template_mcf": "output/nfip_output.tmcf", + "cleaned_csv": "output/nfip_output.csv", + "node_mcf": "output/*.mcf" + } + ], + "source_files": [ + "input_file/fema_nfip_claims.csv" + ], + "cron_schedule": "0 11 1 * *" + } ] - } - +} diff --git a/statvar_imports/finland_census/manifest.json b/statvar_imports/finland_census/manifest.json index 9777b37355..5a76417eb6 100644 --- a/statvar_imports/finland_census/manifest.json +++ b/statvar_imports/finland_census/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output/finland_census_output.tmcf", - "cleaned_csv": "output/finland_census_output.csv" + "cleaned_csv": "output/finland_census_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/france_demographics/manifest.json b/statvar_imports/france_demographics/manifest.json index 85900bd607..37f591d5f9 100644 --- a/statvar_imports/france_demographics/manifest.json +++ b/statvar_imports/france_demographics/manifest.json @@ -13,7 +13,8 @@ "import_inputs": [ { "template_mcf": "output/population_sex_age_groups_output.tmcf", - "cleaned_csv": "output/*_output.csv" + "cleaned_csv": "output/*_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/gdp_by_county_metro_and_other_areas/beagdpv2/manifest.json b/statvar_imports/gdp_by_county_metro_and_other_areas/beagdpv2/manifest.json index 1e8cd07f34..0c6d9d6fd6 100644 --- a/statvar_imports/gdp_by_county_metro_and_other_areas/beagdpv2/manifest.json +++ b/statvar_imports/gdp_by_county_metro_and_other_areas/beagdpv2/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output_files/bea_gdp_output.tmcf", - "cleaned_csv": "output_files/bea_gdp_output.csv" + "cleaned_csv": "output_files/bea_gdp_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 07 * * 2" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/google_sustainability_financial_incentives/manifest.json b/statvar_imports/google_sustainability_financial_incentives/manifest.json index e240268b57..1a7ec22ca7 100644 --- a/statvar_imports/google_sustainability_financial_incentives/manifest.json +++ b/statvar_imports/google_sustainability_financial_incentives/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "financial_incentives_final.tmcf", - "cleaned_csv": "output/financial_incentives.csv" + "cleaned_csv": "output/financial_incentives.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -25,4 +26,4 @@ "cron_schedule": "0 06 * * 3" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/india_ndap/india_nss_health_ailments/manifest.json b/statvar_imports/india_ndap/india_nss_health_ailments/manifest.json index a23cb809d2..712992473d 100644 --- a/statvar_imports/india_ndap/india_nss_health_ailments/manifest.json +++ b/statvar_imports/india_ndap/india_nss_health_ailments/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/india_nss_health_ailments_output.tmcf", - "cleaned_csv": "output/india_nss_health_ailments_output.csv" + "cleaned_csv": "output/india_nss_health_ailments_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index a3e926acf9..1067502742 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/India_LifeExpectancy.tmcf", - "cleaned_csv": "output/India_LifeExpectancy.csv" + "cleaned_csv": "output/India_LifeExpectancy.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "00 11 1,15 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/india_nfhs/manifest.json b/statvar_imports/india_nfhs/manifest.json index d1f40cee66..fc89903e28 100644 --- a/statvar_imports/india_nfhs/manifest.json +++ b/statvar_imports/india_nfhs/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "output/NFHS4_output.tmcf", - "cleaned_csv": "output/*.csv" + "cleaned_csv": "output/*.csv", + "node_mcf": "output/*.mcf" } ], "source_file": [ diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index 50323af5b7..b10af366fe 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output/state_domestic_product_output.tmcf", - "cleaned_csv": "output/state_domestic_product_output.csv" + "cleaned_csv": "output/state_domestic_product_output.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 09 4,18 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/inpe_fire/manifest.json b/statvar_imports/inpe_fire/manifest.json index 271f22d59d..3778857d5a 100644 --- a/statvar_imports/inpe_fire/manifest.json +++ b/statvar_imports/inpe_fire/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output_inpe.tmcf", - "cleaned_csv": "output_inpe.csv" + "cleaned_csv": "output_inpe.csv", + "node_mcf": "*.mcf" } ], "source_files": [ @@ -24,4 +25,4 @@ "cleanup_gcs_volume_mount": false } ] -} \ No newline at end of file +} diff --git a/statvar_imports/ipeds/student_to_faculty_ratio/manifest.json b/statvar_imports/ipeds/student_to_faculty_ratio/manifest.json index 072dc0ca2d..ecba87d2a0 100644 --- a/statvar_imports/ipeds/student_to_faculty_ratio/manifest.json +++ b/statvar_imports/ipeds/student_to_faculty_ratio/manifest.json @@ -8,18 +8,21 @@ "provenance_url": "https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx?gotoReportId=7&fromIpeds=true&sid=859539d0-db09-4d03-a36a-eabf93d07355&rtid=7", "provenance_description": "Integrated Postsecondary Education Data System (IPEDS) is the official online home for the primary federal source of data on U.S. colleges, universities, and technical/vocational institutions.. This import focuses on Student to Faculty Ratio data.", "scripts": [ - "download.py","preprocess.py","run.sh" - ], + "download.py", + "preprocess.py", + "run.sh" + ], "import_inputs": [ { "template_mcf": "processed_output/student_faculty_ratio.tmcf", - "cleaned_csv": "processed_output/*.csv" + "cleaned_csv": "processed_output/*.csv", + "node_mcf": "processed_output/*.mcf" } - ], + ], "source_files": [ "input_files/*.csv" ], "cron_schedule": "0 0 15 7 *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/ireland_census/manifest.json b/statvar_imports/ireland_census/manifest.json index 02fde73912..882f569fdb 100644 --- a/statvar_imports/ireland_census/manifest.json +++ b/statvar_imports/ireland_census/manifest.json @@ -18,50 +18,61 @@ "import_inputs": [ { "template_mcf": "output_files/irl_birth_output.tmcf", - "cleaned_csv": "output_files/irl_birth_output.csv" + "cleaned_csv": "output_files/irl_birth_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_death_output.tmcf", - "cleaned_csv": "output_files/irl_death_output.csv" + "cleaned_csv": "output_files/irl_death_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_external_cause_of_death_output.tmcf", - "cleaned_csv": "output_files/irl_external_cause_of_death_output.csv" + "cleaned_csv": "output_files/irl_external_cause_of_death_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_life_expectancy_output.tmcf", - "cleaned_csv": "output_files/irl_life_expectancy_output.csv" + "cleaned_csv": "output_files/irl_life_expectancy_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_suicide_output.tmcf", - "cleaned_csv": "output_files/irl_suicide_output.csv" + "cleaned_csv": "output_files/irl_suicide_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_aa1_aa2_suicide_output.tmcf", - "cleaned_csv": "output_files/irl_aa1_aa2_suicide_output.csv" + "cleaned_csv": "output_files/irl_aa1_aa2_suicide_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_population_by_age_gender_output.tmcf", - "cleaned_csv": "output_files/irl_population_by_age_gender_output.csv" + "cleaned_csv": "output_files/irl_population_by_age_gender_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_population_by_religion_output.tmcf", - "cleaned_csv": "output_files/irl_population_by_religion_output.csv" + "cleaned_csv": "output_files/irl_population_by_religion_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_population_at_each_census_output.tmcf", - "cleaned_csv": "output_files/irl_population_at_each_census_output.csv" + "cleaned_csv": "output_files/irl_population_at_each_census_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_population_by_gender_output.tmcf", - "cleaned_csv": "output_files/irl_population_by_gender_output.csv" + "cleaned_csv": "output_files/irl_population_by_gender_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/irl_aa1_aa2_employment_output.tmcf", - "cleaned_csv": "output_files/irl_aa1_aa2_employment_output.csv" + "cleaned_csv": "output_files/irl_aa1_aa2_employment_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 05 * * 5" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/mexico_subnational_population_statistics/mexico_census_aa2/manifest.json b/statvar_imports/mexico_subnational_population_statistics/mexico_census_aa2/manifest.json index c7523be11e..3d65fa769e 100644 --- a/statvar_imports/mexico_subnational_population_statistics/mexico_census_aa2/manifest.json +++ b/statvar_imports/mexico_subnational_population_statistics/mexico_census_aa2/manifest.json @@ -19,18 +19,21 @@ "import_inputs": [ { "template_mcf": "output_files/mexico_output_aa0.tmcf", - "cleaned_csv": "output_files/mexico_output_aa0.csv" + "cleaned_csv": "output_files/mexico_output_aa0.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/mexico_output_aa1.tmcf", - "cleaned_csv": "output_files/mexico_output_aa1.csv" + "cleaned_csv": "output_files/mexico_output_aa1.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/mexico_output_aa2.tmcf", - "cleaned_csv": "output_files/mexico_output_aa2.csv" + "cleaned_csv": "output_files/mexico_output_aa2.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 07 * * 3" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/mongolia_imports/mongolia_demographics/manifest.json b/statvar_imports/mongolia_imports/mongolia_demographics/manifest.json index c99782c5a7..cccfaa6ee3 100644 --- a/statvar_imports/mongolia_imports/mongolia_demographics/manifest.json +++ b/statvar_imports/mongolia_imports/mongolia_demographics/manifest.json @@ -1,46 +1,57 @@ { - "import_specifications": [ - { - "import_name": "Mongolia_Demographics", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.1212.mn/en/statcate", - "provenance_description": "Data related to the Mongolia's population and households at country level", - "scripts": ["../common_download_script.py", "run.sh"], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/mid_year_total_population_by_region_output.tmcf", - "cleaned_csv": "output_files/mid_year_total_population_by_region_output.csv" - }, - { - "template_mcf": "output_files/number_of_households_by_region_and_urban_rural_output.tmcf", - "cleaned_csv": "output_files/number_of_households_by_region_and_urban_rural_output.csv" - }, - { - "template_mcf": "output_files/number_of_households_by_region_output.tmcf", - "cleaned_csv": "output_files/number_of_households_by_region_output.csv" - }, - { - "template_mcf": "output_files/resident_population_by_agegroup_15_and_over_and_maritalstatus_output.tmcf", - "cleaned_csv": "output_files/resident_population_by_agegroup_15_and_over_and_maritalstatus_output.csv" - }, - { - "template_mcf": "output_files/total_population_by_age_group_and_sex_output.tmcf", - "cleaned_csv": "output_files/total_population_by_age_group_and_sex_output.csv" - }, - { - "template_mcf": "output_files/total_population_by_region_and_urban_rural_output.tmcf", - "cleaned_csv": "output_files/total_population_by_region_and_urban_rural_output.csv" - }, - { - "template_mcf": "output_files/total_population_by_sex_and_urban_rural_output.tmcf", - "cleaned_csv": "output_files/total_population_by_sex_and_urban_rural_output.csv" + "import_name": "Mongolia_Demographics", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.1212.mn/en/statcate", + "provenance_description": "Data related to the Mongolia's population and households at country level", + "scripts": [ + "../common_download_script.py", + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/mid_year_total_population_by_region_output.tmcf", + "cleaned_csv": "output_files/mid_year_total_population_by_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_households_by_region_and_urban_rural_output.tmcf", + "cleaned_csv": "output_files/number_of_households_by_region_and_urban_rural_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_households_by_region_output.tmcf", + "cleaned_csv": "output_files/number_of_households_by_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/resident_population_by_agegroup_15_and_over_and_maritalstatus_output.tmcf", + "cleaned_csv": "output_files/resident_population_by_agegroup_15_and_over_and_maritalstatus_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/total_population_by_age_group_and_sex_output.tmcf", + "cleaned_csv": "output_files/total_population_by_age_group_and_sex_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/total_population_by_region_and_urban_rural_output.tmcf", + "cleaned_csv": "output_files/total_population_by_region_and_urban_rural_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/total_population_by_sex_and_urban_rural_output.tmcf", + "cleaned_csv": "output_files/total_population_by_sex_and_urban_rural_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 04 * * 6" } - ], - "cron_schedule": "0 04 * * 6" - } - ] + ] } - diff --git a/statvar_imports/mongolia_imports/mongolia_education/manifest.json b/statvar_imports/mongolia_imports/mongolia_education/manifest.json index 4f48270eaa..bff45c248a 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/manifest.json +++ b/statvar_imports/mongolia_imports/mongolia_education/manifest.json @@ -1,41 +1,52 @@ { - "import_specifications": [ - { - "import_name": "Mongolia_Education", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.1212.mn/en/statcate", - "provenance_description": "Data related to the Mongolia's education at country level", - "scripts": ["../common_download_script.py", "run.sh"], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/students_of_universities_and_colleges_by_professional_field_output.tmcf", - "cleaned_csv": "output_files/students_of_universities_and_colleges_by_professional_field_output.csv" - }, - { - "template_mcf": "output_files/students_in_teritary_educational_institutions_by_sex_and_educational_degree_output.tmcf", - "cleaned_csv": "output_files/students_in_teritary_educational_institutions_by_sex_and_educational_degree_output.csv" - }, - { - "template_mcf": "output_files/number_of_students_in_universities_and_colleges_by_region_output.tmcf", - "cleaned_csv": "output_files/number_of_students_in_universities_and_colleges_by_region_output.csv" - }, - { - "template_mcf": "output_files/number_of_kindergartens_by_region_output.tmcf", - "cleaned_csv": "output_files/number_of_kindergartens_by_region_output.csv" - }, - { - "template_mcf": "output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output.tmcf", - "cleaned_csv": "output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output.csv" - }, - { - "template_mcf": "output_files/graduates_of_universities_and_colleges_by_professional_field_output.tmcf", - "cleaned_csv": "output_files/graduates_of_universities_and_colleges_by_professional_field_output.csv" + "import_name": "Mongolia_Education", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.1212.mn/en/statcate", + "provenance_description": "Data related to the Mongolia's education at country level", + "scripts": [ + "../common_download_script.py", + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/students_of_universities_and_colleges_by_professional_field_output.tmcf", + "cleaned_csv": "output_files/students_of_universities_and_colleges_by_professional_field_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/students_in_teritary_educational_institutions_by_sex_and_educational_degree_output.tmcf", + "cleaned_csv": "output_files/students_in_teritary_educational_institutions_by_sex_and_educational_degree_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_students_in_universities_and_colleges_by_region_output.tmcf", + "cleaned_csv": "output_files/number_of_students_in_universities_and_colleges_by_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_kindergartens_by_region_output.tmcf", + "cleaned_csv": "output_files/number_of_kindergartens_by_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output.tmcf", + "cleaned_csv": "output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/graduates_of_universities_and_colleges_by_professional_field_output.tmcf", + "cleaned_csv": "output_files/graduates_of_universities_and_colleges_by_professional_field_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 04 * * 6" } - ], - "cron_schedule": "0 04 * * 6" - } - ] + ] } diff --git a/statvar_imports/mongolia_imports/mongolia_employment/manifest.json b/statvar_imports/mongolia_imports/mongolia_employment/manifest.json index 5fdcef0a5b..555ed9053e 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/manifest.json +++ b/statvar_imports/mongolia_imports/mongolia_employment/manifest.json @@ -1,42 +1,51 @@ { - "import_specifications": [ - { - "import_name": "Mongolia_Employment", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.1212.mn/en/statcate", - "provenance_description": "Data related to the Mongolia's labour and business at country level", - "scripts": ["run.sh"], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/employment_by_classification_of_economic_activities_region_gender_and_agegroup_output.tmcf", - "cleaned_csv": "output_files/employment_by_classification_of_economic_activities_region_gender_and_agegroup_output.csv" - }, - { - "template_mcf": "output_files/employment_by_occupation_by_region_gender_and_agegroup_output.tmcf", - "cleaned_csv": "output_files/employment_by_occupation_by_region_gender_and_agegroup_output.csv" - }, - { - "template_mcf": "output_files/employment_to_population_ratio_by_region_gender_and_agegroup_output.tmcf", - "cleaned_csv": "output_files/employment_to_population_ratio_by_region_gender_and_agegroup_output.csv" - }, - { - "template_mcf": "output_files/labour_force_by_region_gender_and_agegroup_output.tmcf", - "cleaned_csv": "output_files/labour_force_by_region_gender_and_agegroup_output.csv" - }, - { - "template_mcf": "output_files/labour_underutilization_by_region_gender_and_agegroup_output.tmcf", - "cleaned_csv": "output_files/labour_underutilization_by_region_gender_and_agegroup_output.csv" - }, - { - "template_mcf": "output_files/registered_unemployed_by_education_level_region_gender_month_output.tmcf", - "cleaned_csv": "output_files/registered_unemployed_by_education_level_region_gender_month_output.csv" + "import_name": "Mongolia_Employment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.1212.mn/en/statcate", + "provenance_description": "Data related to the Mongolia's labour and business at country level", + "scripts": [ + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/employment_by_classification_of_economic_activities_region_gender_and_agegroup_output.tmcf", + "cleaned_csv": "output_files/employment_by_classification_of_economic_activities_region_gender_and_agegroup_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/employment_by_occupation_by_region_gender_and_agegroup_output.tmcf", + "cleaned_csv": "output_files/employment_by_occupation_by_region_gender_and_agegroup_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/employment_to_population_ratio_by_region_gender_and_agegroup_output.tmcf", + "cleaned_csv": "output_files/employment_to_population_ratio_by_region_gender_and_agegroup_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/labour_force_by_region_gender_and_agegroup_output.tmcf", + "cleaned_csv": "output_files/labour_force_by_region_gender_and_agegroup_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/labour_underutilization_by_region_gender_and_agegroup_output.tmcf", + "cleaned_csv": "output_files/labour_underutilization_by_region_gender_and_agegroup_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/registered_unemployed_by_education_level_region_gender_month_output.tmcf", + "cleaned_csv": "output_files/registered_unemployed_by_education_level_region_gender_month_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 02 * * 6" } - ], - "cron_schedule": "0 02 * * 6" - } - ] + ] } - diff --git a/statvar_imports/mongolia_imports/mongolia_health/manifest.json b/statvar_imports/mongolia_imports/mongolia_health/manifest.json index 0a95769284..c5e695df4f 100644 --- a/statvar_imports/mongolia_imports/mongolia_health/manifest.json +++ b/statvar_imports/mongolia_imports/mongolia_health/manifest.json @@ -1,42 +1,52 @@ { - "import_specifications": [ - { - "import_name": "Mongolia_Health", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.1212.mn/en/statcate", - "provenance_description": "Statistical data related to births and deaths in Mongolia", - "scripts": ["../common_download_script.py", "run.sh"], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/deaths_by_month_and_region_output.tmcf", - "cleaned_csv": "output_files/deaths_by_month_and_region_output.csv" - }, - { - "template_mcf": "output_files/infant_mortality_per_1000_live_births_by_month_region_output.tmcf", - "cleaned_csv": "output_files/infant_mortality_per_1000_live_births_by_month_region_output.csv" - }, - { - "template_mcf": "output_files/live_births_by_month_region_output.tmcf", - "cleaned_csv": "output_files/live_births_by_month_region_output.csv" - }, - { - "template_mcf": "output_files/number_of_abortions_by_region_output.tmcf", - "cleaned_csv": "output_files/number_of_abortions_by_region_output.csv" - }, - { - "template_mcf": "output_files/number_of_hospital_beds_by_type_output.tmcf", - "cleaned_csv": "output_files/number_of_hospital_beds_by_type_output.csv" - }, - { - "template_mcf": "output_files/number_of_mothers_delivered_child_by_month_region_output.tmcf", - "cleaned_csv": "output_files/number_of_mothers_delivered_child_by_month_region_output.csv" + "import_name": "Mongolia_Health", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.1212.mn/en/statcate", + "provenance_description": "Statistical data related to births and deaths in Mongolia", + "scripts": [ + "../common_download_script.py", + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/deaths_by_month_and_region_output.tmcf", + "cleaned_csv": "output_files/deaths_by_month_and_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/infant_mortality_per_1000_live_births_by_month_region_output.tmcf", + "cleaned_csv": "output_files/infant_mortality_per_1000_live_births_by_month_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/live_births_by_month_region_output.tmcf", + "cleaned_csv": "output_files/live_births_by_month_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_abortions_by_region_output.tmcf", + "cleaned_csv": "output_files/number_of_abortions_by_region_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_hospital_beds_by_type_output.tmcf", + "cleaned_csv": "output_files/number_of_hospital_beds_by_type_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/number_of_mothers_delivered_child_by_month_region_output.tmcf", + "cleaned_csv": "output_files/number_of_mothers_delivered_child_by_month_region_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 03 * * 6" } - ], - "cron_schedule": "0 03 * * 6" - } - ] + ] } - diff --git a/statvar_imports/ncses/ncses_demographics_seh_import/manifest.json b/statvar_imports/ncses/ncses_demographics_seh_import/manifest.json index dc38ab6de7..19c13a4f54 100644 --- a/statvar_imports/ncses/ncses_demographics_seh_import/manifest.json +++ b/statvar_imports/ncses/ncses_demographics_seh_import/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/ncses_demographics_seh_import.tmcf", - "cleaned_csv": "output/ncses_demographics_seh_import.csv" + "cleaned_csv": "output/ncses_demographics_seh_import.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 05 12,28 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/ncses/ncses_median_annual_salary/manifest.json b/statvar_imports/ncses/ncses_median_annual_salary/manifest.json index ade1fd7080..5fd57f818e 100644 --- a/statvar_imports/ncses/ncses_median_annual_salary/manifest.json +++ b/statvar_imports/ncses/ncses_median_annual_salary/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/ncses_median_annual_salary.tmcf", - "cleaned_csv": "output/ncses_median_annual_salary.csv" + "cleaned_csv": "output/ncses_median_annual_salary.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 05 13,27 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/ncses/research_doctorate_recipients/manifest.json b/statvar_imports/ncses/research_doctorate_recipients/manifest.json index e78e951b33..ddb218e798 100644 --- a/statvar_imports/ncses/research_doctorate_recipients/manifest.json +++ b/statvar_imports/ncses/research_doctorate_recipients/manifest.json @@ -15,11 +15,13 @@ "import_inputs": [ { "template_mcf": "output_path/research_doctorate_male.tmcf", - "cleaned_csv": "output_path/research_doctorate_male.csv" + "cleaned_csv": "output_path/research_doctorate_male.csv", + "node_mcf": "output_path/*.mcf" }, { "template_mcf": "output_path/research_doctorate_female.tmcf", - "cleaned_csv": "output_path/research_doctorate_female.csv" + "cleaned_csv": "output_path/research_doctorate_female.csv", + "node_mcf": "output_path/*.mcf" } ], "source_files": [ @@ -28,4 +30,4 @@ "cron_schedule": "30 05 12,26 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 2906cb50c3..9abfe3e8d3 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -18,14 +18,16 @@ "import_inputs": [ { "template_mcf": "output_files/ntia_output.tmcf", - "cleaned_csv": "output_files/ntia_output.csv" + "cleaned_csv": "output_files/ntia_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", - "cleaned_csv": "output_files/ntia_age_output.csv" + "cleaned_csv": "output_files/ntia_age_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 5" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index ebb9339cd5..34808d59f4 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -1,24 +1,27 @@ { - "import_specifications": [ - { - "import_name": "NYU_Tennessee_Diabetes", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", - "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", - "scripts": ["download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" - ] , - "source_files": [ - "input_files/*.xlsx" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/tennessee_output.tmcf", - "cleaned_csv": "output_files/tennessee_output.csv" + "import_name": "NYU_Tennessee_Diabetes", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", + "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", + "scripts": [ + "download.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" + ], + "source_files": [ + "input_files/*.xlsx" + ], + "import_inputs": [ + { + "template_mcf": "output_files/tennessee_output.tmcf", + "cleaned_csv": "output_files/tennessee_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 05 * * 5" } - ], - "cron_schedule": "0 05 * * 5" - } - ] + ] } - diff --git a/statvar_imports/nyu_diabetes/texas/manifest.json b/statvar_imports/nyu_diabetes/texas/manifest.json index a46cf39645..ae272c1914 100644 --- a/statvar_imports/nyu_diabetes/texas/manifest.json +++ b/statvar_imports/nyu_diabetes/texas/manifest.json @@ -1,26 +1,27 @@ { - "import_specifications": [ - { - "import_name": "US_Texas_Diabetes", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://healthdata.dshs.texas.gov/dashboard/births-and-deaths/deaths", - "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", - "scripts": [ - "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" - ], - "source_files": [ - "input_files/*.xlsx" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/tennessee_output.tmcf", - "cleaned_csv": "output_files/tennessee_output.csv" + "import_name": "US_Texas_Diabetes", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://healthdata.dshs.texas.gov/dashboard/births-and-deaths/deaths", + "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", + "scripts": [ + "download.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" + ], + "source_files": [ + "input_files/*.xlsx" + ], + "import_inputs": [ + { + "template_mcf": "output_files/tennessee_output.tmcf", + "cleaned_csv": "output_files/tennessee_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 05 * * 5" } - ], - "cron_schedule": "0 05 * * 5" - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/oecd/regional_education/manifest.json b/statvar_imports/oecd/regional_education/manifest.json index 520c86b868..faa9e6390f 100644 --- a/statvar_imports/oecd/regional_education/manifest.json +++ b/statvar_imports/oecd/regional_education/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "output/oecd_regional_education.tmcf", - "cleaned_csv": "output/oecd_regional_education.csv" + "cleaned_csv": "output/oecd_regional_education.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -24,4 +25,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/statvar_imports/oecd/wastewater_treatment/manifest.json b/statvar_imports/oecd/wastewater_treatment/manifest.json index cd3821ddbc..859b1d1832 100644 --- a/statvar_imports/oecd/wastewater_treatment/manifest.json +++ b/statvar_imports/oecd/wastewater_treatment/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "output/oecd_wastewatertreatment.tmcf", - "cleaned_csv": "output/oecd_wastewatertreatment.csv" + "cleaned_csv": "output/oecd_wastewatertreatment.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", @@ -24,4 +25,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/statvar_imports/opendataforafrica/kenya_census/manifest.json b/statvar_imports/opendataforafrica/kenya_census/manifest.json index cc47a1b7ac..050a8d0bde 100644 --- a/statvar_imports/opendataforafrica/kenya_census/manifest.json +++ b/statvar_imports/opendataforafrica/kenya_census/manifest.json @@ -25,47 +25,58 @@ "import_inputs": [ { "template_mcf": "output/dlrrjxg.tmcf", - "cleaned_csv": "output/dlrrjxg.csv" + "cleaned_csv": "output/dlrrjxg.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/egdxgkd.tmcf", - "cleaned_csv": "output/egdxgkd.csv" + "cleaned_csv": "output/egdxgkd.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/emxkej.tmcf", - "cleaned_csv": "output/emxkej.csv" + "cleaned_csv": "output/emxkej.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/fwjfdnc.tmcf", - "cleaned_csv": "output/fwjfdnc.csv" + "cleaned_csv": "output/fwjfdnc.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/ixdvqrf.tmcf", - "cleaned_csv": "output/ixdvqrf.csv" + "cleaned_csv": "output/ixdvqrf.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/rsfzlbg.tmcf", - "cleaned_csv": "output/rsfzlbg.csv" + "cleaned_csv": "output/rsfzlbg.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/srricmg.tmcf", - "cleaned_csv": "output/srricmg.csv" + "cleaned_csv": "output/srricmg.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/tdxdksf.tmcf", - "cleaned_csv": "output/tdxdksf.csv" + "cleaned_csv": "output/tdxdksf.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/vdbvyfd.tmcf", - "cleaned_csv": "output/vdbvyfd.csv" + "cleaned_csv": "output/vdbvyfd.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/welrttb.tmcf", - "cleaned_csv": "output/welrttb.csv" + "cleaned_csv": "output/welrttb.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/xszlbb.tmcf", - "cleaned_csv": "output/xszlbb.csv" + "cleaned_csv": "output/xszlbb.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -74,4 +85,4 @@ "cron_schedule": "30 05 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/opendataforafrica/rwanda_census/manifest.json b/statvar_imports/opendataforafrica/rwanda_census/manifest.json index 81284d023b..448a52451d 100644 --- a/statvar_imports/opendataforafrica/rwanda_census/manifest.json +++ b/statvar_imports/opendataforafrica/rwanda_census/manifest.json @@ -25,43 +25,53 @@ "import_inputs": [ { "template_mcf": "gcs_output/output/edvtatd.tmcf", - "cleaned_csv": "gcs_output/output/edvtatd.csv" + "cleaned_csv": "gcs_output/output/edvtatd.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/eoefmac.tmcf", - "cleaned_csv": "gcs_output/output/eoefmac.csv" + "cleaned_csv": "gcs_output/output/eoefmac.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/hivlaeg.tmcf", - "cleaned_csv": "gcs_output/output/hivlaeg.csv" + "cleaned_csv": "gcs_output/output/hivlaeg.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/kbadgkf.tmcf", - "cleaned_csv": "gcs_output/output/kbadgkf.csv" + "cleaned_csv": "gcs_output/output/kbadgkf.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/mnoirpd.tmcf", - "cleaned_csv": "gcs_output/output/mnoirpd.csv" + "cleaned_csv": "gcs_output/output/mnoirpd.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/shdirje.tmcf", - "cleaned_csv": "gcs_output/output/shdirje.csv" + "cleaned_csv": "gcs_output/output/shdirje.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/uymcezb.tmcf", - "cleaned_csv": "gcs_output/output/uymcezb.csv" + "cleaned_csv": "gcs_output/output/uymcezb.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/vafqyfg.tmcf", - "cleaned_csv": "gcs_output/output/vafqyfg.csv" + "cleaned_csv": "gcs_output/output/vafqyfg.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/ztaoyl.tmcf", - "cleaned_csv": "gcs_output/output/ztaoyl.csv" + "cleaned_csv": "gcs_output/output/ztaoyl.csv", + "node_mcf": "gcs_output/output/*.mcf" }, { "template_mcf": "gcs_output/output/qavhudd.tmcf", - "cleaned_csv": "gcs_output/output/qavhudd.csv" + "cleaned_csv": "gcs_output/output/qavhudd.csv", + "node_mcf": "gcs_output/output/*.mcf" } ], "source_files": [ @@ -70,4 +80,4 @@ "cron_schedule": "30 05 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/school_algebra1/manifest.json b/statvar_imports/school_algebra1/manifest.json index 85148336ef..21844165fd 100644 --- a/statvar_imports/school_algebra1/manifest.json +++ b/statvar_imports/school_algebra1/manifest.json @@ -1,32 +1,34 @@ { - "import_specifications": [ - { - "import_name": "US_UrbanSchool_Algebra1", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://ocrdata.ed.gov/data", - "provenance_description": "This dataset from the Civil Rights Data Collection (CRDC) contains information about student enrollment and passing rates in Algebra 1 courses in US schools, disaggregated by grade level, race, gender, and other demographics.", - "scripts": [ - "download_script.py", - "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.xlsx,input_files/*algebra_i.csv,input_files/*school_data.csv --pv_map=Algebra1_pvmap.csv --config_file=Algebra1_metadata.csv --output_path=output/algebra1_output_ --log_level=-2 --log_every_n=1000" - ], - "source_files": [ - "input_files/*.xlsx","input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/algebra1_output_.tmcf", - "cleaned_csv": "output/algebra1_output_.csv" + "import_name": "US_UrbanSchool_Algebra1", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ocrdata.ed.gov/data", + "provenance_description": "This dataset from the Civil Rights Data Collection (CRDC) contains information about student enrollment and passing rates in Algebra 1 courses in US schools, disaggregated by grade level, race, gender, and other demographics.", + "scripts": [ + "download_script.py", + "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.xlsx,input_files/*algebra_i.csv,input_files/*school_data.csv --pv_map=Algebra1_pvmap.csv --config_file=Algebra1_metadata.csv --output_path=output/algebra1_output_ --log_level=-2 --log_every_n=1000" + ], + "source_files": [ + "input_files/*.xlsx", + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output/algebra1_output_.tmcf", + "cleaned_csv": "output/algebra1_output_.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "30 08 25 * *", + "resource_limits": { + "cpu": 8, + "memory": 256, + "disk": 200 + }, + "validation_config_file": "validation_config.json" } - ], - "cron_schedule": "30 08 25 * *", - "resource_limits": { - "cpu": 8, - "memory": 256, - "disk": 200 - }, - "validation_config_file": "validation_config.json" - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/school_finance/manifest.json b/statvar_imports/school_finance/manifest.json index 65dea70c4f..e7ca303afe 100644 --- a/statvar_imports/school_finance/manifest.json +++ b/statvar_imports/school_finance/manifest.json @@ -1,31 +1,32 @@ { - "import_specifications": [ - { - "import_name": "US_UrbanSchool_Finances", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://ocrdata.ed.gov/data", - "provenance_description": "School Finance dataset contains financial and identifying information for educational institutions, including details on salaries and expenditures for teachers.", - "scripts": [ - "download_script.py", - "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/* --pv_map=school_finance_pvmap.csv --config_file=school_finance_metadata.csv --output_path=output/school_finance_output" - ], - "source_files": [ - "input_files/*" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/school_finance_output.tmcf", - "cleaned_csv": "output/school_finance_output.csv" + "import_name": "US_UrbanSchool_Finances", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ocrdata.ed.gov/data", + "provenance_description": "School Finance dataset contains financial and identifying information for educational institutions, including details on salaries and expenditures for teachers.", + "scripts": [ + "download_script.py", + "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/* --pv_map=school_finance_pvmap.csv --config_file=school_finance_metadata.csv --output_path=output/school_finance_output" + ], + "source_files": [ + "input_files/*" + ], + "import_inputs": [ + { + "template_mcf": "output/school_finance_output.tmcf", + "cleaned_csv": "output/school_finance_output.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "30 08 25 * *", + "resource_limits": { + "cpu": 16, + "memory": 128, + "disk": 500 + } } - ], - "cron_schedule": "30 08 25 * *", - "resource_limits": { - "cpu": 16, - "memory": 128, - "disk": 500 - } - } - ] + ] } diff --git a/statvar_imports/school_retention/manifest.json b/statvar_imports/school_retention/manifest.json index d146d26c90..2294e31326 100644 --- a/statvar_imports/school_retention/manifest.json +++ b/statvar_imports/school_retention/manifest.json @@ -1,31 +1,32 @@ { - "import_specifications": [ - { - "import_name": "US_UrbanSchool_Retention", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://ocrdata.ed.gov/data", - "provenance_description": "This data provides an overview of student retention rates, categorized by gender, race/ethnicity, disability status, and language fluency, specifically for School Grades 1 to 12.", - "scripts": [ - "download_script.py", - "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/* --pv_map=retention_pvmap.csv --config_file=retention_metadata.csv --output_path=output/retention_output" - ], - "source_files": [ - "input_files/*" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/retention_output.tmcf", - "cleaned_csv": "output/retention_output.csv" + "import_name": "US_UrbanSchool_Retention", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ocrdata.ed.gov/data", + "provenance_description": "This data provides an overview of student retention rates, categorized by gender, race/ethnicity, disability status, and language fluency, specifically for School Grades 1 to 12.", + "scripts": [ + "download_script.py", + "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/* --pv_map=retention_pvmap.csv --config_file=retention_metadata.csv --output_path=output/retention_output" + ], + "source_files": [ + "input_files/*" + ], + "import_inputs": [ + { + "template_mcf": "output/retention_output.tmcf", + "cleaned_csv": "output/retention_output.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "30 07 27 * *", + "resource_limits": { + "cpu": 16, + "memory": 128, + "disk": 500 + } } - ], - "cron_schedule": "30 07 27 * *", - "resource_limits": { - "cpu": 16, - "memory": 128, - "disk": 500 - } - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/singapore_census/manifest.json b/statvar_imports/singapore_census/manifest.json index f5dafb07b5..0d74c715f3 100644 --- a/statvar_imports/singapore_census/manifest.json +++ b/statvar_imports/singapore_census/manifest.json @@ -20,7 +20,8 @@ "import_inputs": [ { "template_mcf": "common_tmcf.tmcf", - "cleaned_csv": "output_path/*.csv" + "cleaned_csv": "output_path/*.csv", + "node_mcf": "output_path/*.mcf" } ], "source_files": [ @@ -29,4 +30,4 @@ "cron_schedule": "30 05 14,28 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/southkorea_statistics/demographics/manifest.json b/statvar_imports/southkorea_statistics/demographics/manifest.json index bed2532180..a68692b3ad 100644 --- a/statvar_imports/southkorea_statistics/demographics/manifest.json +++ b/statvar_imports/southkorea_statistics/demographics/manifest.json @@ -1,26 +1,33 @@ { - "import_specifications": [ - { - "import_name": "SouthKorea_Demographics", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", - "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the demographic statistics of South Korea", - "scripts": ["../run.sh gs://unresolved_mcf/country/southkorea/demographics/source_files/", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/population_data.csv --pv_map=population_pvmap.csv --config_file=population_metadata.csv --places_resolved_csv=population_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/population_density_data.csv --pv_map=population_density_pvmap.csv --config_file=population_density_metadata.csv --places_resolved_csv=population_density_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population_density" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/population.tmcf", - "cleaned_csv": "output/population.csv" - }, - { - "template_mcf": "output/population_density.tmcf", - "cleaned_csv": "output/population_density.csv" + "import_name": "SouthKorea_Demographics", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", + "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the demographic statistics of South Korea", + "scripts": [ + "../run.sh gs://unresolved_mcf/country/southkorea/demographics/source_files/", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/population_data.csv --pv_map=population_pvmap.csv --config_file=population_metadata.csv --places_resolved_csv=population_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/population_density_data.csv --pv_map=population_density_pvmap.csv --config_file=population_density_metadata.csv --places_resolved_csv=population_density_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population_density" + ], + "import_inputs": [ + { + "template_mcf": "output/population.tmcf", + "cleaned_csv": "output/population.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/population_density.tmcf", + "cleaned_csv": "output/population_density.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "0 10 1,15 * *", + "source_files": [ + "source_files/*.csv" + ] } - ], - "cron_schedule": "0 10 1,15 * *", - "source_files": ["source_files/*.csv"] - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/southkorea_statistics/education/manifest.json b/statvar_imports/southkorea_statistics/education/manifest.json index c1d7b06f44..274e4dd36f 100644 --- a/statvar_imports/southkorea_statistics/education/manifest.json +++ b/statvar_imports/southkorea_statistics/education/manifest.json @@ -1,61 +1,75 @@ { - "import_specifications": [ - { - "import_name": "SouthKorea_Education", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", - "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the education statistics of South Korea", - "scripts": ["../run.sh gs://unresolved_mcf/country/southkorea/education/input_files/", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Elementary_School_20250821185604.csv --pv_map='elementary_school_pvmap.csv,places_resolved.csv' --config_file=elementary_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/elementary_school", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821182014.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_1", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821184419.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_2", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821184720.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_3", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Junior_College_20250822040439.csv --pv_map='junior_college_pvmap.csv,places_resolved.csv' --config_file=junior_college_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/junior_college_1", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Junior_College_20250822040842.csv --pv_map='junior_college_pvmap.csv,places_resolved.csv' --config_file=junior_college_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/junior_college_2", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Kindergarten_20250821210214.csv --pv_map='kindergarten_pvmap.csv,places_resolved.csv' --config_file=kindergarten_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/kindergarten", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Middle_School_20250821210556.csv --pv_map='middle_school_pvmap.csv,places_resolved.csv' --config_file=middle_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/middle_school_1", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Middle_School_20250821210706.csv --pv_map='middle_school_pvmap.csv,places_resolved.csv' --config_file=middle_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/middle_school_2" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/elementary_school.tmcf", - "cleaned_csv": "output/elementary_school.csv" - }, - { - "template_mcf": "output/high_school_1.tmcf", - "cleaned_csv": "output/high_school_1.csv" - }, - { - "template_mcf": "output/high_school_2.tmcf", - "cleaned_csv": "output/high_school_2.csv" - }, - { - "template_mcf": "output/high_school_3.tmcf", - "cleaned_csv": "output/high_school_3.csv" - }, - { - "template_mcf": "output/junior_college_1.tmcf", - "cleaned_csv": "output/junior_college_1.csv" - }, - { - "template_mcf": "output/junior_college_2.tmcf", - "cleaned_csv": "output/junior_college_2.csv" - }, - { - "template_mcf": "output/kindergarten.tmcf", - "cleaned_csv": "output/kindergarten.csv" - }, - { - "template_mcf": "output/middle_school_1.tmcf", - "cleaned_csv": "output/middle_school_1.csv" - }, - { - "template_mcf": "output/middle_school_2.tmcf", - "cleaned_csv": "output/middle_school_2.csv" + "import_name": "SouthKorea_Education", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", + "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the education statistics of South Korea", + "scripts": [ + "../run.sh gs://unresolved_mcf/country/southkorea/education/input_files/", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Elementary_School_20250821185604.csv --pv_map='elementary_school_pvmap.csv,places_resolved.csv' --config_file=elementary_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/elementary_school", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821182014.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_1", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821184419.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_2", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_High_School_20250821184720.csv --pv_map='high_school_pvmap.csv,places_resolved.csv' --config_file=high_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/high_school_3", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Junior_College_20250822040439.csv --pv_map='junior_college_pvmap.csv,places_resolved.csv' --config_file=junior_college_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/junior_college_1", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Junior_College_20250822040842.csv --pv_map='junior_college_pvmap.csv,places_resolved.csv' --config_file=junior_college_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/junior_college_2", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Kindergarten_20250821210214.csv --pv_map='kindergarten_pvmap.csv,places_resolved.csv' --config_file=kindergarten_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/kindergarten", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Middle_School_20250821210556.csv --pv_map='middle_school_pvmap.csv,places_resolved.csv' --config_file=middle_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/middle_school_1", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/Summary_of_Middle_School_20250821210706.csv --pv_map='middle_school_pvmap.csv,places_resolved.csv' --config_file=middle_school_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/middle_school_2" + ], + "import_inputs": [ + { + "template_mcf": "output/elementary_school.tmcf", + "cleaned_csv": "output/elementary_school.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/high_school_1.tmcf", + "cleaned_csv": "output/high_school_1.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/high_school_2.tmcf", + "cleaned_csv": "output/high_school_2.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/high_school_3.tmcf", + "cleaned_csv": "output/high_school_3.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/junior_college_1.tmcf", + "cleaned_csv": "output/junior_college_1.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/junior_college_2.tmcf", + "cleaned_csv": "output/junior_college_2.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/kindergarten.tmcf", + "cleaned_csv": "output/kindergarten.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/middle_school_1.tmcf", + "cleaned_csv": "output/middle_school_1.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/middle_school_2.tmcf", + "cleaned_csv": "output/middle_school_2.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "0 10 1,15 * *", + "source_files": [ + "source_files/*.csv" + ] } - ], - "cron_schedule": "0 10 1,15 * *", - "source_files": ["source_files/*.csv"] - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/southkorea_statistics/employment/manifest.json b/statvar_imports/southkorea_statistics/employment/manifest.json index 22d73aedcf..f0b3ee724b 100644 --- a/statvar_imports/southkorea_statistics/employment/manifest.json +++ b/statvar_imports/southkorea_statistics/employment/manifest.json @@ -1,36 +1,45 @@ { - "import_specifications": [ - { - "import_name": "SouthKorea_Employment", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", - "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the employment statistics of South Korea", - "scripts": ["../run.sh gs://unresolved_mcf/country/southkorea/employment/source_files/", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/employmentstatus_data.csv --pv_map=employmentstatus_pvmap.csv --config_file=employmentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/employmentstatus", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/sexandeducationalattainment_unemploymentstatus_data.csv --pv_map=sexandeducationalattainment_unemploymentstatus_pvmap.csv --config_file=sexandeducationalattainment_unemploymentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/sexandeducationalattainment_unemploymentstatus", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/unemploymentrate_by_gender_age_data.csv --pv_map=unemploymentrate_by_gender_age_pvmap.csv --config_file=unemploymentrate_by_gender_age_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/unemploymentrate_by_gender_age", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/unemploymentstatus_data.csv --pv_map=unemploymentstatus_pvmap.csv --config_file=unemploymentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/unemploymentstatus" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/employmentstatus.tmcf", - "cleaned_csv": "output/employmentstatus.csv" - }, - { - "template_mcf": "output/sexandeducationalattainment_unemploymentstatus.tmcf", - "cleaned_csv": "output/sexandeducationalattainment_unemploymentstatus.csv" - }, - { - "template_mcf": "output/unemploymentrate_by_gender_age.tmcf", - "cleaned_csv": "output/unemploymentrate_by_gender_age.csv" - }, - { - "template_mcf": "output/unemploymentstatus.tmcf", - "cleaned_csv": "output/unemploymentstatus.csv" + "import_name": "SouthKorea_Employment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", + "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the employment statistics of South Korea", + "scripts": [ + "../run.sh gs://unresolved_mcf/country/southkorea/employment/source_files/", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/employmentstatus_data.csv --pv_map=employmentstatus_pvmap.csv --config_file=employmentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/employmentstatus", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/sexandeducationalattainment_unemploymentstatus_data.csv --pv_map=sexandeducationalattainment_unemploymentstatus_pvmap.csv --config_file=sexandeducationalattainment_unemploymentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/sexandeducationalattainment_unemploymentstatus", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/unemploymentrate_by_gender_age_data.csv --pv_map=unemploymentrate_by_gender_age_pvmap.csv --config_file=unemploymentrate_by_gender_age_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/unemploymentrate_by_gender_age", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/unemploymentstatus_data.csv --pv_map=unemploymentstatus_pvmap.csv --config_file=unemploymentstatus_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/unemploymentstatus" + ], + "import_inputs": [ + { + "template_mcf": "output/employmentstatus.tmcf", + "cleaned_csv": "output/employmentstatus.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/sexandeducationalattainment_unemploymentstatus.tmcf", + "cleaned_csv": "output/sexandeducationalattainment_unemploymentstatus.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/unemploymentrate_by_gender_age.tmcf", + "cleaned_csv": "output/unemploymentrate_by_gender_age.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/unemploymentstatus.tmcf", + "cleaned_csv": "output/unemploymentstatus.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "0 10 1,15 * *", + "source_files": [ + "source_files/*.csv" + ] } - ], - "cron_schedule": "0 10 1,15 * *", - "source_files": ["source_files/*.csv"] - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/southkorea_statistics/health/manifest.json b/statvar_imports/southkorea_statistics/health/manifest.json index 57226853db..f0bf0c2443 100644 --- a/statvar_imports/southkorea_statistics/health/manifest.json +++ b/statvar_imports/southkorea_statistics/health/manifest.json @@ -1,26 +1,33 @@ { - "import_specifications": [ - { - "import_name": "SouthKorea_Health", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", - "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the health statistics of South Korea", - "scripts": ["../run.sh gs://unresolved_mcf/country/southkorea/health/source_files/", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/health_data.csv --pv_map=health_pvmap.csv --config_file=health_metadata.csv --places_resolved_csv=health_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/health", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/diseasereport_data.csv --pv_map=diseasereport_pvmap.csv --config_file=diseasereport_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/diseasereport" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output/health.tmcf", - "cleaned_csv": "output/health.csv" - }, - { - "template_mcf": "output/diseasereport.tmcf", - "cleaned_csv": "output/diseasereport.csv" + "import_name": "SouthKorea_Health", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://kosis.kr/eng/statisticsList/statisticsListIndex.do", + "provenance_description": "This data is from the South Korea Statistics Agency (KOSA), that describes about the health statistics of South Korea", + "scripts": [ + "../run.sh gs://unresolved_mcf/country/southkorea/health/source_files/", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/health_data.csv --pv_map=health_pvmap.csv --config_file=health_metadata.csv --places_resolved_csv=health_places_resolved_csv.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/health", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/diseasereport_data.csv --pv_map=diseasereport_pvmap.csv --config_file=diseasereport_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/diseasereport" + ], + "import_inputs": [ + { + "template_mcf": "output/health.tmcf", + "cleaned_csv": "output/health.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/diseasereport.tmcf", + "cleaned_csv": "output/diseasereport.csv", + "node_mcf": "output/*.mcf" + } + ], + "cron_schedule": "0 10 1,15 * *", + "source_files": [ + "source_files/*.csv" + ] } - ], - "cron_schedule": "0 10 1,15 * *", - "source_files": ["source_files/*.csv"] - } - ] -} \ No newline at end of file + ] +} diff --git a/statvar_imports/statistics_new_zealand/new_zealand_census/manifest.json b/statvar_imports/statistics_new_zealand/new_zealand_census/manifest.json index 09f568b834..a06db3bf31 100644 --- a/statvar_imports/statistics_new_zealand/new_zealand_census/manifest.json +++ b/statvar_imports/statistics_new_zealand/new_zealand_census/manifest.json @@ -18,14 +18,16 @@ "import_inputs": [ { "template_mcf": "output_files/nzl_national_output.tmcf", - "cleaned_csv": "output_files/nzl_national_output.csv" + "cleaned_csv": "output_files/nzl_national_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/nzl_subnational_output.tmcf", - "cleaned_csv": "output_files/nzl_subnational_output.csv" + "cleaned_csv": "output_files/nzl_subnational_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 2" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 4d2b33ca93..0e017d9eb7 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -18,7 +18,8 @@ { "template_mcf": "StatisticsPoland_output.tmcf", "cleaned_csv": "StatisticsPoland_output.csv", - "stat_var_mcf": "StatisticsPoland_output_stat_vars.mcf" + "stat_var_mcf": "StatisticsPoland_output_stat_vars.mcf", + "node_mcf": "*.mcf" } ], "cron_schedule": "0 0 1 1,4,7,10 *" diff --git a/statvar_imports/tuberculosis_percentage/manifest.json b/statvar_imports/tuberculosis_percentage/manifest.json index 3c0cbb381e..4145895a4a 100644 --- a/statvar_imports/tuberculosis_percentage/manifest.json +++ b/statvar_imports/tuberculosis_percentage/manifest.json @@ -18,7 +18,8 @@ { "template_mcf": "output_files/tuberculosisPercentage_output.tmcf", "cleaned_csv": "output_files/tuberculosisPercentage_output.csv", - "stat_var_mcf": "output_files/tuberculosisPercentage_output_stat_vars.mcf" + "stat_var_mcf": "output_files/tuberculosisPercentage_output_stat_vars.mcf", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 0 1 1,4,7,10 *" diff --git a/statvar_imports/uae_bayanat/uae_population/manifest.json b/statvar_imports/uae_bayanat/uae_population/manifest.json index 4b2ec79027..6ed0643b32 100644 --- a/statvar_imports/uae_bayanat/uae_population/manifest.json +++ b/statvar_imports/uae_bayanat/uae_population/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output/uae_population_output.tmcf", - "cleaned_csv": "output/uae_population_output.csv" + "cleaned_csv": "output/uae_population_output.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 05 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/undata/manifest.json b/statvar_imports/undata/manifest.json index 055dd20efd..7e06be2869 100644 --- a/statvar_imports/undata/manifest.json +++ b/statvar_imports/undata/manifest.json @@ -18,10 +18,11 @@ "import_inputs": [ { "template_mcf": "output/UNData.tmcf", - "cleaned_csv": "output/filter_undata.csv" + "cleaned_csv": "output/filter_undata.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 05 * * 3" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_bachelors_degree_data/nces_bachelors_degree_by_field_import/manifest.json b/statvar_imports/us_bachelors_degree_data/nces_bachelors_degree_by_field_import/manifest.json index a32fcbb1da..859e7c6955 100644 --- a/statvar_imports/us_bachelors_degree_data/nces_bachelors_degree_by_field_import/manifest.json +++ b/statvar_imports/us_bachelors_degree_data/nces_bachelors_degree_by_field_import/manifest.json @@ -22,22 +22,26 @@ "import_inputs": [ { "template_mcf": "output_files/nces_female_output.tmcf", - "cleaned_csv": "output_files/nces_female_output.csv" + "cleaned_csv": "output_files/nces_female_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/nces_male_output.tmcf", - "cleaned_csv": "output_files/nces_male_output.csv" + "cleaned_csv": "output_files/nces_male_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/nces_female_output_latest.tmcf", - "cleaned_csv": "output_files/nces_female_output_latest.csv" + "cleaned_csv": "output_files/nces_female_output_latest.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/nces_male_output_latest.tmcf", - "cleaned_csv": "output_files/nces_male_output_latest.csv" + "cleaned_csv": "output_files/nces_male_output_latest.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 05 * * 4" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_bls/bls_ces/manifest.json b/statvar_imports/us_bls/bls_ces/manifest.json index 7f76ad25dc..e076b9290e 100644 --- a/statvar_imports/us_bls/bls_ces/manifest.json +++ b/statvar_imports/us_bls/bls_ces/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/bls_ces.tmcf", - "cleaned_csv": "output/bls_ces.csv" + "cleaned_csv": "output/bls_ces.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 06 * * 1" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_bls/bls_ces_state/manifest.json b/statvar_imports/us_bls/bls_ces_state/manifest.json index 89c8c5148e..f78d938124 100644 --- a/statvar_imports/us_bls/bls_ces_state/manifest.json +++ b/statvar_imports/us_bls/bls_ces_state/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/bls_ces_state.tmcf", - "cleaned_csv": "output/bls_ces_state.csv" + "cleaned_csv": "output/bls_ces_state.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 06 * * 3" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_bls/cpi_category/manifest.json b/statvar_imports/us_bls/cpi_category/manifest.json index 615704d52d..b0ffeb590a 100644 --- a/statvar_imports/us_bls/cpi_category/manifest.json +++ b/statvar_imports/us_bls/cpi_category/manifest.json @@ -16,7 +16,8 @@ "import_inputs": [ { "template_mcf": "output/cpi_u/cpi_u.tmcf", - "cleaned_csv": "output/*/*.csv" + "cleaned_csv": "output/*/*.csv", + "node_mcf": "output/*/*.mcf" } ], "source_files": [ @@ -26,4 +27,4 @@ "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_bls/us_cpi/manifest.json b/statvar_imports/us_bls/us_cpi/manifest.json index d34d215aab..3a77ea620e 100644 --- a/statvar_imports/us_bls/us_cpi/manifest.json +++ b/statvar_imports/us_bls/us_cpi/manifest.json @@ -15,7 +15,8 @@ "import_inputs": [ { "template_mcf": "output/cpi_w.tmcf", - "cleaned_csv": "output/*.csv" + "cleaned_csv": "output/*.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -29,4 +30,4 @@ "cron_schedule": "0 10 21 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_cdc/single_race/manifest.json b/statvar_imports/us_cdc/single_race/manifest.json index 9808a6c1c5..a0490c5ed7 100644 --- a/statvar_imports/us_cdc/single_race/manifest.json +++ b/statvar_imports/us_cdc/single_race/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/underlyingcauseofdeath2018_2023singlerace.tmcf", - "cleaned_csv": "output/underlyingcauseofdeath2018_2023singlerace.csv" + "cleaned_csv": "output/underlyingcauseofdeath2018_2023singlerace.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 08 8-14 * 6" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_census/pep/us_census_pep_asrh/manifest.json b/statvar_imports/us_census/pep/us_census_pep_asrh/manifest.json index cf2a4d7ac4..69fcf8053b 100644 --- a/statvar_imports/us_census/pep/us_census_pep_asrh/manifest.json +++ b/statvar_imports/us_census/pep/us_census_pep_asrh/manifest.json @@ -16,15 +16,18 @@ "import_inputs": [ { "template_mcf": "gcs_output/output/county/pep_asrh_county.tmcf", - "cleaned_csv": "gcs_output/output/county/pep_asrh_county.csv" + "cleaned_csv": "gcs_output/output/county/pep_asrh_county.csv", + "node_mcf": "gcs_output/output/county/*.mcf" }, { "template_mcf": "gcs_output/output/national/pep_asrh_national.tmcf", - "cleaned_csv": "gcs_output/output/national/pep_asrh_national.csv" + "cleaned_csv": "gcs_output/output/national/pep_asrh_national.csv", + "node_mcf": "gcs_output/output/national/*.mcf" }, { "template_mcf": "gcs_output/output/state/pep_asrh_state.tmcf", - "cleaned_csv": "gcs_output/output/state/pep_asrh_state.csv" + "cleaned_csv": "gcs_output/output/state/pep_asrh_state.csv", + "node_mcf": "gcs_output/output/state/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json index 71a273e9a9..ea7e353183 100644 --- a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json +++ b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/monthly_retail.tmcf", - "cleaned_csv": "output/monthly_retail.csv" + "cleaned_csv": "output/monthly_retail.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/us_crash/fars_crashdata/manifest.json b/statvar_imports/us_crash/fars_crashdata/manifest.json index 6eaf883111..d6db15546a 100644 --- a/statvar_imports/us_crash/fars_crashdata/manifest.json +++ b/statvar_imports/us_crash/fars_crashdata/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "crash_tmcf.tmcf", - "cleaned_csv": "gcs_output/output/*.csv" + "cleaned_csv": "gcs_output/output/*.csv", + "node_mcf": "gcs_output/output/*.mcf" } ], "cron_schedule": "0 05 * * 2" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_education/new_york_education/manifest.json b/statvar_imports/us_education/new_york_education/manifest.json index f0295a3090..a5726447f3 100644 --- a/statvar_imports/us_education/new_york_education/manifest.json +++ b/statvar_imports/us_education/new_york_education/manifest.json @@ -15,11 +15,13 @@ "import_inputs": [ { "template_mcf": "output/english.tmcf", - "cleaned_csv": "output/english.csv" + "cleaned_csv": "output/english.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/math.tmcf", - "cleaned_csv": "output/math.csv" + "cleaned_csv": "output/math.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ diff --git a/statvar_imports/us_federal_reserve_h15_interest_rates/us_federal_rates/manifest.json b/statvar_imports/us_federal_reserve_h15_interest_rates/us_federal_rates/manifest.json index c8c00a538c..0f384381ec 100644 --- a/statvar_imports/us_federal_reserve_h15_interest_rates/us_federal_rates/manifest.json +++ b/statvar_imports/us_federal_reserve_h15_interest_rates/us_federal_rates/manifest.json @@ -18,14 +18,16 @@ "import_inputs": [ { "template_mcf": "output_files/us_fed_interest_rate_output.tmcf", - "cleaned_csv": "output_files/us_fed_interest_rate_output.csv" + "cleaned_csv": "output_files/us_fed_interest_rate_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/us_fed_fund_output.tmcf", - "cleaned_csv": "output_files/us_fed_fund_output.csv" + "cleaned_csv": "output_files/us_fed_fund_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 6" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_hbcu_data/nces_hbcu_enrollment_import/manifest.json b/statvar_imports/us_hbcu_data/nces_hbcu_enrollment_import/manifest.json index 11867c0a7e..53373e51d7 100644 --- a/statvar_imports/us_hbcu_data/nces_hbcu_enrollment_import/manifest.json +++ b/statvar_imports/us_hbcu_data/nces_hbcu_enrollment_import/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output_files/nces_hbcu_output_tabn313.30.tmcf", - "cleaned_csv": "output_files/nces_hbcu_output_tabn313.30.csv" + "cleaned_csv": "output_files/nces_hbcu_output_tabn313.30.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 07 * * 5" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_nces/nces_employed_college_grads/manifest.json b/statvar_imports/us_nces/nces_employed_college_grads/manifest.json index 720b942338..e7c9d86d8d 100644 --- a/statvar_imports/us_nces/nces_employed_college_grads/manifest.json +++ b/statvar_imports/us_nces/nces_employed_college_grads/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output/nces_college.tmcf", - "cleaned_csv": "output/nces_college.csv" + "cleaned_csv": "output/nces_college.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 07 4,18 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_newyork/ny_diabetes/manifest.json b/statvar_imports/us_newyork/ny_diabetes/manifest.json index 77dd74c289..9d1e1676fd 100644 --- a/statvar_imports/us_newyork/ny_diabetes/manifest.json +++ b/statvar_imports/us_newyork/ny_diabetes/manifest.json @@ -5,26 +5,29 @@ "curator_emails": [ "support@datacommons.org" ], - "provenance_url": "https://www.health.ny.gov/statistics/prevention/injury_prevention/information_for_action/", + "provenance_url": "https://www.health.ny.gov/statistics/prevention/injury_prevention/information_for_action/", "provenance_description": "The Behavioral Risk Factor Surveillance System (BRFSS) is an annual statewide telephone survey of adults developed by the Centers for Disease Control and Prevention (CDC) and administered by the New York State Department of Health. This data represents the percentage of adults with diagnosed diabetes by county in New York for the years 2016, 2018, and 2021.", "scripts": [ - "run.sh", + "run.sh", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_2016_data.csv --pv_map=ny_diabetes_pv_map.csv --config_file=ny_diabetes_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=ny_diabetes_place_resolver.csv --output_path=output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2016", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_2018_data.csv --pv_map=ny_diabetes_pv_map.csv --config_file=ny_diabetes_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=ny_diabetes_place_resolver.csv --output_path=output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2018", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_2021_data.csv --pv_map=ny_diabetes_pv_map.csv --config_file=ny_diabetes_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=ny_diabetes_place_resolver.csv --output_path=output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2021" ], "import_inputs": [ - { + { "template_mcf": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2016.tmcf", - "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2016.csv" + "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2016.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2018.tmcf", - "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2018.csv" + "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2018.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2021.tmcf", - "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2021.csv" + "cleaned_csv": "output_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_output_2021.csv", + "node_mcf": "output_files/*.mcf" } ], "source_files": [ @@ -32,7 +35,6 @@ "input_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_2018_data.csv", "input_files/ny_percentage_of_adults_with_diagnosed_diabetes_by_county_2021_data.csv" ] - } ] } diff --git a/statvar_imports/us_steam_degrees_data/nces_stem_degrees_import/manifest.json b/statvar_imports/us_steam_degrees_data/nces_stem_degrees_import/manifest.json index df15fe2b1b..1a82b14aa5 100644 --- a/statvar_imports/us_steam_degrees_data/nces_stem_degrees_import/manifest.json +++ b/statvar_imports/us_steam_degrees_data/nces_stem_degrees_import/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output/nces_steam_degree.tmcf", - "cleaned_csv": "output/nces_steam_degree.csv" + "cleaned_csv": "output/nces_steam_degree.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 05 4,18 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/manifest.json b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/manifest.json index 151a87368a..c15ce62925 100644 --- a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/manifest.json +++ b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "common_output.tmcf", - "cleaned_csv": "output_files/output_*.csv" + "cleaned_csv": "output_files/output_*.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 08 8,23 * *", @@ -28,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/gifted_and_talented/manifest.json b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/gifted_and_talented/manifest.json index f7c2d34471..4aaf061297 100644 --- a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/gifted_and_talented/manifest.json +++ b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/gifted_and_talented/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "common_output.tmcf", - "cleaned_csv": "output_files/output_*.csv" + "cleaned_csv": "output_files/output_*.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 11 8,23 * *", @@ -28,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/international_baccalaureate/manifest.json b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/international_baccalaureate/manifest.json index bd8e41f579..c53bcd8630 100644 --- a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/international_baccalaureate/manifest.json +++ b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/international_baccalaureate/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "common_output.tmcf", - "cleaned_csv": "output_files/output_*.csv" + "cleaned_csv": "output_files/output_*.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 14 8,23 * *", @@ -28,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/covid_directional_indicators/manifest.json b/statvar_imports/us_urban_school/covid_directional_indicators/manifest.json index 6ffe4f5a4a..742aa14789 100644 --- a/statvar_imports/us_urban_school/covid_directional_indicators/manifest.json +++ b/statvar_imports/us_urban_school/covid_directional_indicators/manifest.json @@ -1,34 +1,42 @@ { - "import_specifications": [ - { - "import_name": "US_UrbanSchool_Covid_Directional_Indicators", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://civilrightsdata.ed.gov/data", - "provenance_description": "", - "scripts": ["download.py --data_type=covid","run.sh"], - "source_files": [ - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/covid_directional_indicators_output1.tmcf", - "cleaned_csv": "output_files/covid_directional_indicators_output1.csv" - }, - { - "template_mcf": "output_files/covid_directional_indicators_output2.tmcf", - "cleaned_csv": "output_files/covid_directional_indicators_output2.csv" - }, - { - "template_mcf": "output_files/covid_directional_indicators_output3.tmcf", - "cleaned_csv": "output_files/covid_directional_indicators_output3.csv" - }, - { - "template_mcf": "output_files/covid_directional_indicators_output4.tmcf", - "cleaned_csv": "output_files/covid_directional_indicators_output4.csv" + "import_name": "US_UrbanSchool_Covid_Directional_Indicators", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://civilrightsdata.ed.gov/data", + "provenance_description": "", + "scripts": [ + "download.py --data_type=covid", + "run.sh" + ], + "source_files": [ + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/covid_directional_indicators_output1.tmcf", + "cleaned_csv": "output_files/covid_directional_indicators_output1.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/covid_directional_indicators_output2.tmcf", + "cleaned_csv": "output_files/covid_directional_indicators_output2.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/covid_directional_indicators_output3.tmcf", + "cleaned_csv": "output_files/covid_directional_indicators_output3.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/covid_directional_indicators_output4.tmcf", + "cleaned_csv": "output_files/covid_directional_indicators_output4.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 05 * * 5" } - ], - "cron_schedule": "0 05 * * 5" - } - ] + ] } - diff --git a/statvar_imports/us_urban_school/enrollment/manifest.json b/statvar_imports/us_urban_school/enrollment/manifest.json index d10550cdae..4a48a15e83 100644 --- a/statvar_imports/us_urban_school/enrollment/manifest.json +++ b/statvar_imports/us_urban_school/enrollment/manifest.json @@ -1,55 +1,68 @@ { - "import_specifications": [ - { - "import_name": "US_UrbanSchool_Enrollment", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://civilrightsdata.ed.gov/data", - "provenance_description": "This import utilizes US Urban School Enrollment data for student enrollment counts, broken down by gender, race, Limited English Proficiency, and disability status (Section 504 and Individualized Education Program).", - "scripts": ["../download.py --data_type=enrollment","run.sh"], - "source_files": [ - "input_files/*.xlsx", - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/enrollment_by_race_and_sex_2010_2014_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_race_and_sex_2010_2014_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_race_and_sex_2016_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_race_and_sex_2016_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_race_and_sex_2018_2022_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_race_and_sex_2018_2022_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_disability_and_sex_2010_2014_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2010_2014_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_disability_and_sex_2016_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2016_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_disability_and_sex_2018_2022_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2018_2022_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_lep_and_sex_2010_2014_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2010_2014_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_lep_and_sex_2016_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2016_output.csv" - }, - { - "template_mcf": "output_files/enrollment_by_lep_and_sex_2018_2022_output.tmcf", - "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2018_2022_output.csv" + "import_name": "US_UrbanSchool_Enrollment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://civilrightsdata.ed.gov/data", + "provenance_description": "This import utilizes US Urban School Enrollment data for student enrollment counts, broken down by gender, race, Limited English Proficiency, and disability status (Section 504 and Individualized Education Program).", + "scripts": [ + "../download.py --data_type=enrollment", + "run.sh" + ], + "source_files": [ + "input_files/*.xlsx", + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/enrollment_by_race_and_sex_2010_2014_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_race_and_sex_2010_2014_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_race_and_sex_2016_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_race_and_sex_2016_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_race_and_sex_2018_2022_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_race_and_sex_2018_2022_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_disability_and_sex_2010_2014_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2010_2014_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_disability_and_sex_2016_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2016_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_disability_and_sex_2018_2022_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_disability_and_sex_2018_2022_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_lep_and_sex_2010_2014_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2010_2014_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_lep_and_sex_2016_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2016_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/enrollment_by_lep_and_sex_2018_2022_output.tmcf", + "cleaned_csv": "output_files/enrollment_by_lep_and_sex_2018_2022_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 05 * * 5" } - ], - "cron_schedule": "0 05 * * 5" - } - ] + ] } - diff --git a/statvar_imports/us_urban_school/maths_and_science_enrollment/manifest.json b/statvar_imports/us_urban_school/maths_and_science_enrollment/manifest.json index 947fd4dace..3974d9d7e7 100644 --- a/statvar_imports/us_urban_school/maths_and_science_enrollment/manifest.json +++ b/statvar_imports/us_urban_school/maths_and_science_enrollment/manifest.json @@ -1,60 +1,73 @@ { - "import_specifications": [ - { - "import_name": "US_Urban_MathsAndScience_Course_Enrollment", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://civilrightsdata.ed.gov/data", - "provenance_description": "This import contains data on enrollment in Biology, Chemistry, Advanced Math, Calculus, Algebra II, Physics, and Geometry courses.", - "scripts": ["download.py","run.sh"], - "source_files": [ - "input_files/*.xlsx", - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/maths_and_science_2010_2012_output.tmcf", - "cleaned_csv": "output_files/maths_and_science_2010_2012_output.csv" - }, - { - "template_mcf": "output_files/maths_and_science_2016_output.tmcf", - "cleaned_csv": "output_files/maths_and_science_2016_output.csv" - }, - { - "template_mcf": "output_files/advanced_mathematics_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/advanced_mathematics_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/algebra2_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/algebra2_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/biology_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/biology_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/calculus_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/calculus_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/chemistry_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/chemistry_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/geometry_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/geometry_2018_onwards_output.csv" - }, - { - "template_mcf": "output_files/physics_2018_onwards_output.tmcf", - "cleaned_csv": "output_files/physics_2018_onwards_output.csv" - } - ], - "resource_limits": { + "import_name": "US_Urban_MathsAndScience_Course_Enrollment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://civilrightsdata.ed.gov/data", + "provenance_description": "This import contains data on enrollment in Biology, Chemistry, Advanced Math, Calculus, Algebra II, Physics, and Geometry courses.", + "scripts": [ + "download.py", + "run.sh" + ], + "source_files": [ + "input_files/*.xlsx", + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/maths_and_science_2010_2012_output.tmcf", + "cleaned_csv": "output_files/maths_and_science_2010_2012_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/maths_and_science_2016_output.tmcf", + "cleaned_csv": "output_files/maths_and_science_2016_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/advanced_mathematics_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/advanced_mathematics_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/algebra2_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/algebra2_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/biology_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/biology_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/calculus_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/calculus_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/chemistry_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/chemistry_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/geometry_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/geometry_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/physics_2018_onwards_output.tmcf", + "cleaned_csv": "output_files/physics_2018_onwards_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "resource_limits": { "cpu": 16, "memory": 64, "disk": 500 - }, - "cron_schedule": "0 04 * * 5" - } - ] + }, + "cron_schedule": "0 04 * * 5" + } + ] } - diff --git a/statvar_imports/us_urban_school/sat_act_participation/school/manifest.json b/statvar_imports/us_urban_school/sat_act_participation/school/manifest.json index 995f03deb8..5f360a1305 100644 --- a/statvar_imports/us_urban_school/sat_act_participation/school/manifest.json +++ b/statvar_imports/us_urban_school/sat_act_participation/school/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "common_output.tmcf", - "cleaned_csv": "output_files_school/output_*.csv" + "cleaned_csv": "output_files_school/output_*.csv", + "node_mcf": "output_files_school/*.mcf" } ], "cron_schedule": "0 05 8,23 * *", @@ -28,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/sat_act_participation/state/manifest.json b/statvar_imports/us_urban_school/sat_act_participation/state/manifest.json index 5481e47d73..6485733503 100644 --- a/statvar_imports/us_urban_school/sat_act_participation/state/manifest.json +++ b/statvar_imports/us_urban_school/sat_act_participation/state/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "common_output.tmcf", - "cleaned_csv": "output_files_state/output_*.csv" + "cleaned_csv": "output_files_state/output_*.csv", + "node_mcf": "output_files_state/*.mcf" } ], "cron_schedule": "0 05 8,23 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_urban_school/teachers/manifest.json b/statvar_imports/us_urban_school/teachers/manifest.json index 98ec3106a4..b22e13d3fc 100644 --- a/statvar_imports/us_urban_school/teachers/manifest.json +++ b/statvar_imports/us_urban_school/teachers/manifest.json @@ -1,31 +1,38 @@ { - "import_specifications": [ - { - "import_name": "US_Urban_Schools_Teachers_And_Staff", - "curator_emails": ["support@datacommons.org"], - "provenance_url": "https://civilrightsdata.ed.gov/data", - "provenance_description": "This import contains US data on the number of FTE teachers and staff at each school.", - "scripts": ["../download.py --data_type=teachers","run.sh"], - "source_files": [ - "input_files/*.xlsx", - "input_files/*.csv" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "output_files/teachers_and_staff_2010_2014_output.tmcf", - "cleaned_csv": "output_files/teachers_and_staff_2010_2014_output.csv" - }, - { - "template_mcf": "output_files/teachers_and_staff_2016_output.tmcf", - "cleaned_csv": "output_files/teachers_and_staff_2016_output.csv" - }, - { - "template_mcf": "output_files/teachers_and_staff_2018_2022_output.tmcf", - "cleaned_csv": "output_files/teachers_and_staff_2018_2022_output.csv" + "import_name": "US_Urban_Schools_Teachers_And_Staff", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://civilrightsdata.ed.gov/data", + "provenance_description": "This import contains US data on the number of FTE teachers and staff at each school.", + "scripts": [ + "../download.py --data_type=teachers", + "run.sh" + ], + "source_files": [ + "input_files/*.xlsx", + "input_files/*.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/teachers_and_staff_2010_2014_output.tmcf", + "cleaned_csv": "output_files/teachers_and_staff_2010_2014_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/teachers_and_staff_2016_output.tmcf", + "cleaned_csv": "output_files/teachers_and_staff_2016_output.csv", + "node_mcf": "output_files/*.mcf" + }, + { + "template_mcf": "output_files/teachers_and_staff_2018_2022_output.tmcf", + "cleaned_csv": "output_files/teachers_and_staff_2018_2022_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "cron_schedule": "0 5 1,15 * *" } - ], - "cron_schedule": "0 5 1,15 * *" - } - ] + ] } - diff --git a/statvar_imports/usa_dol/minimum_wage/manifest.json b/statvar_imports/usa_dol/minimum_wage/manifest.json index 399df3d228..16cce22600 100644 --- a/statvar_imports/usa_dol/minimum_wage/manifest.json +++ b/statvar_imports/usa_dol/minimum_wage/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/us_dol_wages.tmcf", - "cleaned_csv": "output/us_dol_wages.csv" + "cleaned_csv": "output/us_dol_wages.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "30 09 * * 5" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/who/adolescent_birth_rate/manifest.json b/statvar_imports/who/adolescent_birth_rate/manifest.json index d044f29c3e..cdaf158cb4 100644 --- a/statvar_imports/who/adolescent_birth_rate/manifest.json +++ b/statvar_imports/who/adolescent_birth_rate/manifest.json @@ -1,35 +1,40 @@ { - "import_specifications": [ - { - "import_name": "Adolescent_Birth_Rate", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://data.who.int/indicators/i/24C65FE/27D371A", - "provenance_description": "The annual number of births to females aged 10-14 or 15-19 years per 1,000 females in the respective age group.", - "scripts": [ - "data_download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/* --pv_map=adolescent_birth_rate_pvmap.csv --config_file=adolescent_birth_rate_metadata.csv --output_path=./adolescent_birth_rate_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" - ], - "import_inputs": [ + "import_specifications": [ { - "template_mcf": "adolescent_birth_rate_output.tmcf", - "cleaned_csv": "adolescent_birth_rate_output.csv" - } - ], - "source_files": [ - "input_files/*.csv" - ], - "cron_schedule": "0 0 20 6 *", - "resource_limits": {"cpu": 4, "memory": 8, "disk":100} + "import_name": "Adolescent_Birth_Rate", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://data.who.int/indicators/i/24C65FE/27D371A", + "provenance_description": "The annual number of births to females aged 10-14 or 15-19 years per 1,000 females in the respective age group.", + "scripts": [ + "data_download.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/* --pv_map=adolescent_birth_rate_pvmap.csv --config_file=adolescent_birth_rate_metadata.csv --output_path=./adolescent_birth_rate_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" + ], + "import_inputs": [ + { + "template_mcf": "adolescent_birth_rate_output.tmcf", + "cleaned_csv": "adolescent_birth_rate_output.csv", + "node_mcf": "*.mcf" + } + ], + "source_files": [ + "input_files/*.csv" + ], + "cron_schedule": "0 0 20 6 *", + "resource_limits": { + "cpu": 4, + "memory": 8, + "disk": 100 + } } ], "config_override": { - "invoke_import_validation": true, - "invoke_import_tool": true, - "invoke_differ_tool": true, - "skip_input_upload": false, - "skip_gcs_upload": false, - "cleanup_gcs_volume_mount": false + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false } } diff --git a/statvar_imports/who_covid19/manifest.json b/statvar_imports/who_covid19/manifest.json index cac29c0367..f403d0633a 100644 --- a/statvar_imports/who_covid19/manifest.json +++ b/statvar_imports/who_covid19/manifest.json @@ -3,13 +3,13 @@ { "import_name": "WHO_COVID19", "curator_emails": [ - "support@datacommons.org" + "support@datacommons.org" ], "provenance_url": "https://data.who.int/dashboards/covid19/cases?n=c", "provenance_description": "World Health Organization's (WHO) raw data and statistical releases covering global COVID-19 cases, deaths, hospitalizations, and vaccination coverage", "scripts": [ "download.sh", - "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/COV_VAC_UPTAKE_2021_2023.csv --pv_map=pvmap/COV_VAC_UPTAKE_2021_2023_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/COV_VAC_UPTAKE_2021_2023", + "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/COV_VAC_UPTAKE_2021_2023.csv --pv_map=pvmap/COV_VAC_UPTAKE_2021_2023_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/COV_VAC_UPTAKE_2021_2023", "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/COV_VAC_UPTAKE_2024.csv --pv_map=pvmap/COV_VAC_UPTAKE_2024_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/COV_VAC_UPTAKE_2024", "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/WHO-COVID-19-global-monthly-death-by-age-data.csv --pv_map=pvmap/WHO-COVID-19-global-monthly-death-by-age-data_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/death_by_age", "../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/WHO-COVID-19-global-daily-data.csv --pv_map=pvmap/WHO-COVID-19-global-daily-data_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --config_file=common_metadata.csv --output_path=output/global_daily_data", @@ -17,26 +17,31 @@ ], "import_inputs": [ { - "template_mcf": "output/COV_VAC_UPTAKE_2021_2023.tmcf", - "cleaned_csv": "output/COV_VAC_UPTAKE_2021_2023.csv" - }, - { - "template_mcf": "output/COV_VAC_UPTAKE_2024.tmcf", - "cleaned_csv": "output/COV_VAC_UPTAKE_2024.csv" - }, - { - "template_mcf": "output/death_by_age.tmcf", - "cleaned_csv": "output/death_by_age.csv" - }, - { - "template_mcf": "output/global_daily_data.tmcf", - "cleaned_csv": "output/global_daily_data.csv" - }, - { - "template_mcf": "output/global_hosp_icu_data.tmcf", - "cleaned_csv": "output/global_hosp_icu_data.csv" - } - ], + "template_mcf": "output/COV_VAC_UPTAKE_2021_2023.tmcf", + "cleaned_csv": "output/COV_VAC_UPTAKE_2021_2023.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/COV_VAC_UPTAKE_2024.tmcf", + "cleaned_csv": "output/COV_VAC_UPTAKE_2024.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/death_by_age.tmcf", + "cleaned_csv": "output/death_by_age.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/global_daily_data.tmcf", + "cleaned_csv": "output/global_daily_data.csv", + "node_mcf": "output/*.mcf" + }, + { + "template_mcf": "output/global_hosp_icu_data.tmcf", + "cleaned_csv": "output/global_hosp_icu_data.csv", + "node_mcf": "output/*.mcf" + } + ], "source_files": [ "input_files/*.csv" ], diff --git a/statvar_imports/who_tuberculosis/treatmentoutcome_tb_hiv/manifest.json b/statvar_imports/who_tuberculosis/treatmentoutcome_tb_hiv/manifest.json index da85791eaa..a4316de53d 100644 --- a/statvar_imports/who_tuberculosis/treatmentoutcome_tb_hiv/manifest.json +++ b/statvar_imports/who_tuberculosis/treatmentoutcome_tb_hiv/manifest.json @@ -14,7 +14,8 @@ "import_inputs": [ { "template_mcf": "output/tuberculosis_hiv_output.tmcf", - "cleaned_csv": "output/tuberculosis_hiv_output.csv" + "cleaned_csv": "output/tuberculosis_hiv_output.csv", + "node_mcf": "output/*.mcf" } ], "source_files": [ @@ -23,4 +24,4 @@ "cron_schedule": "0 10 10,21 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/world_bank/commodity_market/manifest.json b/statvar_imports/world_bank/commodity_market/manifest.json index 165b95f78c..74c7490fbb 100644 --- a/statvar_imports/world_bank/commodity_market/manifest.json +++ b/statvar_imports/world_bank/commodity_market/manifest.json @@ -19,27 +19,33 @@ "import_inputs": [ { "template_mcf": "output/commodity_monthly_price.tmcf", - "cleaned_csv": "output/commodity_monthly_price.csv" + "cleaned_csv": "output/commodity_monthly_price.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/commodity_annual_real_price.tmcf", - "cleaned_csv": "output/commodity_annual_real_price.csv" + "cleaned_csv": "output/commodity_annual_real_price.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/commodity_annual_nominal_price.tmcf", - "cleaned_csv": "output/commodity_annual_nominal_price.csv" + "cleaned_csv": "output/commodity_annual_nominal_price.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/commodity_monthly_indices.tmcf", - "cleaned_csv": "output/commodity_monthly_indices.csv" + "cleaned_csv": "output/commodity_monthly_indices.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/commodity_annual_real_indices.tmcf", - "cleaned_csv": "output/commodity_annual_real_indices.csv" + "cleaned_csv": "output/commodity_annual_real_indices.csv", + "node_mcf": "output/*.mcf" }, { "template_mcf": "output/commodity_annual_nominal_indices.tmcf", - "cleaned_csv": "output/commodity_annual_nominal_indices.csv" + "cleaned_csv": "output/commodity_annual_nominal_indices.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 10 1,15 * *", diff --git a/statvar_imports/world_bank/subnational_demographics_stats/manifest.json b/statvar_imports/world_bank/subnational_demographics_stats/manifest.json index 5b6d9e8a0c..61e83750ef 100644 --- a/statvar_imports/world_bank/subnational_demographics_stats/manifest.json +++ b/statvar_imports/world_bank/subnational_demographics_stats/manifest.json @@ -17,10 +17,11 @@ "import_inputs": [ { "template_mcf": "output_files/wb_subnational_output.tmcf", - "cleaned_csv": "output_files/wb_subnational_output.csv" + "cleaned_csv": "output_files/wb_subnational_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 6" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/world_bank/worldbank_ids/manifest.json b/statvar_imports/world_bank/worldbank_ids/manifest.json index 29070f6f00..2d125a4d10 100644 --- a/statvar_imports/world_bank/worldbank_ids/manifest.json +++ b/statvar_imports/world_bank/worldbank_ids/manifest.json @@ -20,22 +20,26 @@ "import_inputs": [ { "template_mcf": "output/currency/currency.tmcf", - "cleaned_csv": "output/currency/currency.csv" + "cleaned_csv": "output/currency/currency.csv", + "node_mcf": "output/currency/*.mcf" }, { "template_mcf": "output/interest/interest_output.tmcf", - "cleaned_csv": "output/interest/interest_output.csv" + "cleaned_csv": "output/interest/interest_output.csv", + "node_mcf": "output/interest/*.mcf" }, { "template_mcf": "output/principal/principal_output.tmcf", - "cleaned_csv": "output/principal/principal_output.csv" + "cleaned_csv": "output/principal/principal_output.csv", + "node_mcf": "output/principal/*.mcf" }, { "template_mcf": "output/disbursed/disbursed_output.tmcf", - "cleaned_csv": "output/disbursed/disbursed_output.csv" + "cleaned_csv": "output/disbursed/disbursed_output.csv", + "node_mcf": "output/disbursed/*.mcf" } ], "cron_schedule": "0 4 7,23 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/zurich/manifest.json b/statvar_imports/zurich/manifest.json index 9ec063ffb1..01e74aea14 100644 --- a/statvar_imports/zurich/manifest.json +++ b/statvar_imports/zurich/manifest.json @@ -17,7 +17,8 @@ "import_inputs": [ { "template_mcf": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.tmcf", - "cleaned_csv": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.csv" + "cleaned_csv": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.csv", + "node_mcf": "wir_2552_wiki/output/*.mcf" } ], "cron_schedule": "0 2 29 * *" @@ -39,7 +40,8 @@ "import_inputs": [ { "template_mcf": "bev_3903_age10_wiki/output/zurich_population_bev_3903_age10_wiki.tmcf", - "cleaned_csv": "bev_3903_age10_wiki/output/zurich_population_bev_3903_age10_wiki.csv" + "cleaned_csv": "bev_3903_age10_wiki/output/zurich_population_bev_3903_age10_wiki.csv", + "node_mcf": "bev_3903_age10_wiki/output/*.mcf" } ], "cron_schedule": "0 6 29 * *" @@ -61,7 +63,8 @@ "import_inputs": [ { "template_mcf": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.tmcf", - "cleaned_csv": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv" + "cleaned_csv": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv", + "node_mcf": "bev_3240_wiki/output/*.mcf" } ], "cron_schedule": "30 11 29 * *" @@ -83,7 +86,8 @@ "import_inputs": [ { "template_mcf": "bev_4031_hel_wiki/output/zurich_bev_4031_hel_wiki.tmcf", - "cleaned_csv": "bev_4031_hel_wiki/output/zurich_bev_4031_hel_wiki.csv" + "cleaned_csv": "bev_4031_hel_wiki/output/zurich_bev_4031_hel_wiki.csv", + "node_mcf": "bev_4031_hel_wiki/output/*.mcf" } ], "cron_schedule": "45 15 29 * *" @@ -105,7 +109,8 @@ "import_inputs": [ { "template_mcf": "bev_4031_wiki/output/zurich_bev_4031_wiki.tmcf", - "cleaned_csv": "bev_4031_wiki/output/zurich_bev_4031_wiki.csv" + "cleaned_csv": "bev_4031_wiki/output/zurich_bev_4031_wiki.csv", + "node_mcf": "bev_4031_wiki/output/*.mcf" } ], "cron_schedule": "0 20 29 * *" @@ -128,7 +133,8 @@ "import_inputs": [ { "template_mcf": "bev_3903_hel_wiki/output/zurich_population_bev_3903_hel_wiki.tmcf", - "cleaned_csv": "bev_3903_hel_wiki/output/zurich_population_bev_3903_hel_wiki.csv" + "cleaned_csv": "bev_3903_hel_wiki/output/zurich_population_bev_3903_hel_wiki.csv", + "node_mcf": "bev_3903_hel_wiki/output/*.mcf" } ], "cron_schedule": "15 1 29 * *" @@ -150,7 +156,8 @@ "import_inputs": [ { "template_mcf": "bev_4031_sex_wiki/output/zurich_bev_4031_sex_wiki.tmcf", - "cleaned_csv": "bev_4031_sex_wiki/output/zurich_bev_4031_sex_wiki.csv" + "cleaned_csv": "bev_4031_sex_wiki/output/zurich_bev_4031_sex_wiki.csv", + "node_mcf": "bev_4031_sex_wiki/output/*.mcf" } ], "cron_schedule": "07 19 29 * *" @@ -172,10 +179,11 @@ "import_inputs": [ { "template_mcf": "bev_3903_sex_wiki/output/zurich_population_bev_3903_sex_wiki.tmcf", - "cleaned_csv": "bev_3903_sex_wiki/output/zurich_population_bev_3903_sex_wiki.csv" + "cleaned_csv": "bev_3903_sex_wiki/output/zurich_population_bev_3903_sex_wiki.csv", + "node_mcf": "bev_3903_sex_wiki/output/*.mcf" } ], "cron_schedule": "55 10 29 * *" } ] -} \ No newline at end of file +}