diff --git a/cardano_node_tests/tests/issues.py b/cardano_node_tests/tests/issues.py index 1795a5f4c..5121c9cff 100644 --- a/cardano_node_tests/tests/issues.py +++ b/cardano_node_tests/tests/issues.py @@ -145,6 +145,16 @@ fixed_in="13.7.0.3", message="Swapped min_pool_cost / coins_per_utxo_size.", ) +dbsync_2150 = blockers.GH( + issue=2150, + repo="IntersectMBO/cardano-db-sync", + message="only_utxo preset (tx_out bootstrap) inserts 0 tx_out and crashes near tip.", +) +dbsync_2151 = blockers.GH( + issue=2151, + repo="IntersectMBO/cardano-db-sync", + message="only_utxo preset populates governance tables, contrary to the docs.", +) ledger_3731 = blockers.GH( issue=3731, diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index c0f8c55b9..09b18aad1 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -13,9 +13,11 @@ from cardano_node_tests.tests import common from cardano_node_tests.utils import cluster_nodes from cardano_node_tests.utils import configuration +from cardano_node_tests.utils import dbsync_queries from cardano_node_tests.utils import dbsync_service_manager as db_sync from cardano_node_tests.utils import dbsync_utils from cardano_node_tests.utils import helpers +from cardano_node_tests.utils import logfiles LOGGER = logging.getLogger(__name__) @@ -43,8 +45,13 @@ class ColumnCondition(enum.StrEnum): ZERO = "column_condition:=0" IS_NULL = "column_condition:IS NULL" + IS_NOT_NULL = "column_condition:IS NOT NULL" +# On-chain governance tables. Off-chain vote metadata is controlled by `offchain_vote_data` +# and has to be fetched from the anchor URLs (only cluster-local/localhost anchors need +# --allow-private-offchain-urls; public URLs are fetched regardless), so it is covered +# separately by the offchain_vote_data subtests rather than bundled here. GOVERNANCE_TABLES = ( db_sync.Table.COMMITTEE_DE_REGISTRATION, db_sync.Table.COMMITTEE_MEMBER, @@ -56,15 +63,21 @@ class ColumnCondition(enum.StrEnum): db_sync.Table.DREP_REGISTRATION, db_sync.Table.EPOCH_STATE, db_sync.Table.GOV_ACTION_PROPOSAL, + db_sync.Table.VOTING_ANCHOR, + db_sync.Table.VOTING_PROCEDURE, + db_sync.Table.TREASURY_WITHDRAWAL, +) + +# Off-chain vote metadata tables, populated only when `offchain_vote_data` is enabled and +# db-sync is allowed to fetch the (private/localhost) anchor URLs. +OFFCHAIN_VOTE_TABLES = ( db_sync.Table.OFF_CHAIN_VOTE_DATA, db_sync.Table.OFF_CHAIN_VOTE_DREP_DATA, db_sync.Table.OFF_CHAIN_VOTE_EXTERNAL_UPDATE, db_sync.Table.OFF_CHAIN_VOTE_FETCH_ERROR, db_sync.Table.OFF_CHAIN_VOTE_GOV_ACTION_DATA, db_sync.Table.OFF_CHAIN_VOTE_REFERENCE, - db_sync.Table.VOTING_ANCHOR, - db_sync.Table.VOTING_PROCEDURE, - db_sync.Table.TREASURY_WITHDRAWAL, + db_sync.Table.OFF_CHAIN_VOTE_AUTHOR, ) @@ -125,6 +138,32 @@ def check_dbsync_state( raise ValueError(error_msg) +def wait_for_tables_not_empty( + tables: tp.Iterable[str | db_sync.Table], + *, + timeout: int = 600, +) -> None: + """Wait until all given db-sync tables have data. + + Off-chain data (pool/vote metadata) is fetched asynchronously and can appear up to + several minutes after db-sync starts (the fetch loop sleeps ~300s between passes), so + such tables must be polled rather than checked once. Raises ``TimeoutError`` (via + ``retry_query``) if any table is still empty after ``timeout`` seconds. + """ + # Materialize once so a generator argument is not exhausted by the first (failing) + # poll, which would make every subsequent retry see an empty list and pass falsely. + tables = list(tables) + + def _query_func() -> bool: + empty_tables = [table for table in tables if dbsync_utils.table_empty(table=table)] + if empty_tables: + msg = f"Following tables are still empty: {empty_tables}" + raise dbsync_utils.DbSyncNoResponseError(msg) + return True + + dbsync_utils.retry_query(query_func=_query_func, timeout=timeout) + + @pytest.fixture def db_sync_manager( cluster_singleton: clusterlib.ClusterLib, # noqa: ARG001 @@ -159,8 +198,25 @@ class TestDBSyncConfig: def get_subtests(self) -> tp.Generator[tp.Callable]: """Get the DB-Sync Config scenarios. - The scenarios are executed as subtests in the `test_dbsync_config` test. + The scenarios are executed as subtests in the `test_dbsync_config` test, + grouped by the db-sync config option each set exercises. """ + yield from self._subtests_tx_out() + yield from self._subtests_governance() + yield from self._subtests_tx_cbor() + yield from self._subtests_multi_asset() + yield from self._subtests_plutus() + yield from self._subtests_metadata() + yield from self._subtests_shelley() + yield from self._subtests_ledger() + yield from self._subtests_offchain_pool() + yield from self._subtests_offchain_vote() + yield from self._subtests_remove_jsonb() + yield from self._subtests_disable_epoch() + yield from self._subtests_presets() + + def _subtests_tx_out(self) -> tp.Generator[tp.Callable]: + """Subtests for the `tx_out` option (modes, force_tx_in, use_address_table).""" def basic_tx_out( db_sync_manager: db_sync.DBSyncManager, @@ -200,6 +256,80 @@ def basic_tx_out( yield basic_tx_out + def tx_out_consumed( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out=consumed` (no force_tx_in). + + Consumption is tracked via `tx_out.consumed_by_tx_id`, so `tx_in` stays empty + while `tx_out` / `ma_tx_out` are populated. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=False + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.EMPTY, + } + ) + assert dbsync_utils.column_exists( + table=db_sync.Table.TX_OUT, column="consumed_by_tx_id" + ), "`consumed` mode should add the `tx_out.consumed_by_tx_id` column" + + yield tx_out_consumed + + def tx_out_consumed_force_tx_in( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out=consumed` with force_tx_in: `tx_in` is populated alongside `tx_out`.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.NOT_EMPTY, + } + ) + + yield tx_out_consumed_force_tx_in + + def tx_out_use_address_table( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out` with use_address_table: addresses go to a separate `address` table.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.ENABLE, use_address_table=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.ADDRESS: TableCondition.EXISTS, + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + } + ) + assert not dbsync_utils.table_empty(table=db_sync.Table.ADDRESS), ( + "`use_address_table` should populate the `address` table" + ) + + yield tx_out_use_address_table + + def _subtests_governance(self) -> tp.Generator[tp.Callable]: + """Subtests for the `governance` option.""" + def governance( db_sync_manager: db_sync.DBSyncManager, ): @@ -211,18 +341,7 @@ def governance( ) # Off-chain data is inserted into the DB a few minutes after the restart of db-sync - def _query_func(): - empty_tables = [ - table for table in GOVERNANCE_TABLES if dbsync_utils.table_empty(table=table) - ] - - if empty_tables: - msg = f"Following tables are still empty: {empty_tables}" - raise dbsync_utils.DbSyncNoResponseError(msg) - - return True - - dbsync_utils.retry_query(query_func=_query_func, timeout=600) + wait_for_tables_not_empty(GOVERNANCE_TABLES, timeout=600) check_dbsync_state( expected_state={t: TableCondition.NOT_EMPTY for t in GOVERNANCE_TABLES} # noqa: C420 @@ -237,6 +356,9 @@ def _query_func(): yield governance + def _subtests_tx_cbor(self) -> tp.Generator[tp.Callable]: + """Subtests for the `tx_cbor` option.""" + def tx_cbor_value_enable( db_sync_manager: db_sync.DBSyncManager, ): @@ -263,6 +385,9 @@ def tx_cbor_value_disable( yield tx_cbor_value_disable + def _subtests_multi_asset(self) -> tp.Generator[tp.Callable]: + """Subtests for the `multi_asset` option.""" + def multi_asset_enable( db_sync_manager: db_sync.DBSyncManager, ): @@ -289,32 +414,550 @@ def multi_asset_disable( yield multi_asset_disable + def _subtests_plutus(self) -> tp.Generator[tp.Callable]: + """Subtests for the `plutus` option.""" + + def plutus_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `plutus`: script / redeemer / redeemer_data / datum are populated. + + Needs a Plutus tx that locks/spends with a datum on chain. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=True)) + check_dbsync_state( + expected_state={ + db_sync.Table.SCRIPT: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.NOT_EMPTY, + db_sync.Table.DATUM: TableCondition.NOT_EMPTY, + } + ) + + yield plutus_enable + + def plutus_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `plutus`: no script-execution data. + + redeemer / redeemer_data / datum stay empty despite a Plutus tx on chain. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=False)) + check_dbsync_state( + expected_state={ + db_sync.Table.REDEEMER: TableCondition.EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.EMPTY, + db_sync.Table.DATUM: TableCondition.EMPTY, + } + ) + + yield plutus_disable + + def _subtests_metadata(self) -> tp.Generator[tp.Callable]: + """Subtests for the `metadata` option (enable/disable and keys filter).""" + + def metadata_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `metadata`: tx_metadata is populated from txs carrying metadata.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=True)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) + + yield metadata_enable + + def metadata_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `metadata`: tx_metadata stays empty despite a tx with metadata.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=False)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.EMPTY}) + + yield metadata_disable + + def metadata_keys_filter( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `metadata.keys` filter: only metadata with the listed key is stored.""" + keep_key = 2 + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_metadata(enable=True, keys=[keep_key]) + ) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) + # Every stored metadata row must have the single kept key. + dbsync_utils.check_column_condition( + table=db_sync.Table.TX_METADATA, column="key", condition=f"= {keep_key}" + ) + + yield metadata_keys_filter + + def _subtests_shelley(self) -> tp.Generator[tp.Callable]: + """Subtests for the `shelley` option (enable/disable side effects).""" + + def shelley_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `shelley`: certificate data (stake_registration, pool_update) is set.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=True)) + check_dbsync_state( + expected_state={ + db_sync.Table.STAKE_REGISTRATION: TableCondition.NOT_EMPTY, + db_sync.Table.POOL_UPDATE: TableCondition.NOT_EMPTY, + } + ) + + yield shelley_enable + + def shelley_disable_independence( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `shelley` is independent of `ledger`. + + `epoch_stake` is ledger-controlled, so it stays populated with shelley off. + Certificate tables are not asserted empty: tx-era certs are gated by `shelley`, + but genesis pool/stake registrations are inserted unconditionally + (Shelley/Genesis.hs), so `pool_update` / `stake_registration` keep genesis rows. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=False)) + check_dbsync_state( + expected_state={ + db_sync.Table.EPOCH_STAKE: TableCondition.NOT_EMPTY, + } + ) + + yield shelley_disable_independence + + def _subtests_ledger(self) -> tp.Generator[tp.Callable]: + """Subtests for the `ledger` modes and `pool_stat` (ledger-derived data).""" + # Tables populated only from ledger state; empty unless `ledger` maintains and uses it. + ledger_derived_tables = ( + db_sync.Table.REWARD, + db_sync.Table.EPOCH_STAKE, + db_sync.Table.ADA_POTS, + db_sync.Table.EPOCH_PARAM, + ) + + def ledger_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `ledger`: ledger-derived tables and columns are populated. + + reward / epoch_stake / ada_pots / epoch_param, redeemer.fee and tx deposits. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.ENABLE) + ) + check_dbsync_state( + expected_state={ + **dict.fromkeys(ledger_derived_tables, TableCondition.NOT_EMPTY), + db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NOT_NULL, + } + ) + # With ledger state, deposits are computed: at least one tx has a positive deposit. + assert ( + dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") > 0 + ), "ledger=enable should record ledger-derived (positive) tx deposits" + + yield ledger_enable + + def ledger_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `ledger`: derived tables empty, redeemer.fee null, no deposits.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.DISABLE) + ) + check_dbsync_state( + expected_state={ + **dict.fromkeys(ledger_derived_tables, TableCondition.EMPTY), + db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NULL, + } + ) + # No ledger state -> no positive deposits (tx.deposit isn't uniformly NULL; some + # txs keep 0, so assert the meaningful effect: no positive deposit remains). + assert ( + dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") == 0 + ), "ledger=disable should drop ledger-derived (positive) tx deposits" + + yield ledger_disable + + def ledger_ignore( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `ledger=ignore`: state is kept but unused, so derived tables stay empty.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.IGNORE) + ) + check_dbsync_state( + expected_state=dict.fromkeys(ledger_derived_tables, TableCondition.EMPTY) + ) + + yield ledger_ignore + + def pool_stat_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `pool_stat`: per-epoch pool stats stored after an epoch boundary.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_pool_stat(value=db_sync.SettingState.ENABLE) + ) + check_dbsync_state(expected_state={db_sync.Table.POOL_STAT: TableCondition.NOT_EMPTY}) + + yield pool_stat_enable + + def pool_stat_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `pool_stat` option.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_pool_stat(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state(expected_state={db_sync.Table.POOL_STAT: TableCondition.EMPTY}) + + yield pool_stat_disable + + def _subtests_offchain_pool(self) -> tp.Generator[tp.Callable]: + """Subtests for `offchain_pool_data` (needs --allow-private-offchain-urls).""" + + def offchain_pool_data_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `offchain_pool_data`: pool metadata is fetched into off_chain_pool_data. + + Fetch is async (~300s loop) so the table is polled. Skipped without private URLs + allowed or when no pool metadata is registered on chain. + """ + if not dbsync_utils.allow_private_offchain_urls_enabled(): + pytest.skip("requires db-sync started with --allow-private-offchain-urls") + + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_offchain_pool_data(value=db_sync.SettingState.ENABLE) + ) + if dbsync_utils.table_empty(table=db_sync.Table.POOL_METADATA_REF): + pytest.skip("no pool metadata registered on chain to fetch") + wait_for_tables_not_empty([db_sync.Table.OFF_CHAIN_POOL_DATA], timeout=600) + # `off_chain_pool_fetch_error` is intentionally not asserted empty: in a full run + # other pools can have unreachable metadata, and db-sync may record a transient + # fetch error before a later successful retry populates the data. + check_dbsync_state( + expected_state={db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.NOT_EMPTY} + ) + + yield offchain_pool_data_enable + + def offchain_pool_data_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `offchain_pool_data`: no fetch. + + off_chain_pool_data / off_chain_pool_fetch_error stay empty; the on-chain + pool_metadata_ref is still recorded. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_offchain_pool_data(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.EMPTY, + db_sync.Table.OFF_CHAIN_POOL_FETCH_ERROR: TableCondition.EMPTY, + db_sync.Table.POOL_METADATA_REF: TableCondition.NOT_EMPTY, + } + ) + + yield offchain_pool_data_disable + + def _subtests_offchain_vote(self) -> tp.Generator[tp.Callable]: + """Subtests for `offchain_vote_data` (needs --allow-private-offchain-urls).""" + + def offchain_vote_data_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `offchain_vote_data=disable` gates the fetch independently of `governance`. + + With governance on but vote data off, voting_anchor is recorded but no anchor + metadata is fetched, so all off_chain_vote_* tables stay empty. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_governance( + value=db_sync.SettingState.ENABLE + ).with_offchain_vote_data(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.VOTING_ANCHOR: TableCondition.NOT_EMPTY, + **dict.fromkeys(OFFCHAIN_VOTE_TABLES, TableCondition.EMPTY), + } + ) + + yield offchain_vote_data_disable + + def offchain_vote_data_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `offchain_vote_data`: anchor metadata is fetched. + + The fetch result lands in off_chain_vote_data (or off_chain_vote_fetch_error). + Skipped without private URLs allowed or with no fetchable vote anchor on chain. + is_valid and the CIP sub-tables need conformant anchors; asserting those (using + the #3497 anchor vectors) is left to a dedicated off-chain test. + """ + if not dbsync_utils.allow_private_offchain_urls_enabled(): + pytest.skip("requires db-sync started with --allow-private-offchain-urls") + if ( + dbsync_queries.query_rows_count( + table="voting_anchor", column="url", condition="!= ''" + ) + == 0 + ): + pytest.skip("no fetchable governance vote anchors on chain") + + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_governance( + value=db_sync.SettingState.ENABLE + ).with_offchain_vote_data(value=db_sync.SettingState.ENABLE) + ) + + # The fetch is asynchronous; wait until db-sync has recorded the result either as + # fetched data or as a fetch error. + def _query_func() -> bool: + data = not dbsync_utils.table_empty(table=db_sync.Table.OFF_CHAIN_VOTE_DATA) + err = not dbsync_utils.table_empty(table=db_sync.Table.OFF_CHAIN_VOTE_FETCH_ERROR) + if not (data or err): + msg = "off_chain_vote_data / off_chain_vote_fetch_error still empty" + raise dbsync_utils.DbSyncNoResponseError(msg) + return True + + dbsync_utils.retry_query(query_func=_query_func, timeout=600) + + yield offchain_vote_data_enable + + def _subtests_remove_jsonb(self) -> tp.Generator[tp.Callable]: + """Subtests for `remove_jsonb_from_schema` column-type effects.""" + # jsonb columns controlled by remove_jsonb_from_schema (excluding *.json columns, which + # `json_type` also governs). Column types are schema-level, so row counts don't matter. + jsonb_columns = ( + (db_sync.Table.DATUM, "value"), + (db_sync.Table.COST_MODEL, "costs"), + (db_sync.Table.GOV_ACTION_PROPOSAL, "description"), + ) + + def remove_jsonb_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `remove_jsonb_from_schema`: jsonb columns keep the jsonb type.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_remove_jsonb_from_schema( + value=db_sync.SettingState.DISABLE + ) + ) + for table, column in jsonb_columns: + assert dbsync_utils.column_data_type(table=table, column=column) == "jsonb", ( + f"{table}.{column} should be jsonb when remove_jsonb_from_schema is disabled" + ) + + yield remove_jsonb_disable + + def remove_jsonb_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `remove_jsonb_from_schema`: jsonb columns drop the jsonb type.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_remove_jsonb_from_schema( + value=db_sync.SettingState.ENABLE + ) + ) + for table, column in jsonb_columns: + dtype = dbsync_utils.column_data_type(table=table, column=column) + assert dtype is not None and dtype != "jsonb", ( + f"{table}.{column} should not be jsonb when remove_jsonb_from_schema is " + f"enabled (got {dtype})" + ) + + yield remove_jsonb_enable + + def _subtests_disable_epoch(self) -> tp.Generator[tp.Callable]: + """Subtests for the `disable_epoch` option (controls the `epoch` rollup view).""" + + def disable_epoch_true( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `disable_epoch=true`: the `epoch` view returns no rows.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_disable_epoch(value=True) + ) + check_dbsync_state(expected_state={db_sync.View.EPOCH: TableCondition.EMPTY}) + + yield disable_epoch_true + + def disable_epoch_false( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `disable_epoch=false`: the `epoch` view is populated.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_disable_epoch(value=False) + ) + check_dbsync_state(expected_state={db_sync.View.EPOCH: TableCondition.NOT_EMPTY}) + + yield disable_epoch_false + + def _subtests_presets(self) -> tp.Generator[tp.Callable]: + """Subtests for insert-option presets (exercise db-sync's own preset expansion).""" + + def preset_full( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `full` preset: all insert options on except tx_cbor and off-chain.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.FULL) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.NOT_EMPTY, + db_sync.Table.TX_CBOR: TableCondition.EMPTY, + } + ) + + yield preset_full + + def preset_only_utxo( + db_sync_manager: db_sync.DBSyncManager, # noqa: ARG001 + ): + """Test the `only_utxo` preset (docs: block/tx/tx_out/ma_tx_out only). + + Skipped: the preset uses tx_out bootstrap mode, under which db-sync inserts 0 + tx_out and then crashes near tip (dbsync #2150). The crash exits the dbsync + service and that unexpected exit is logged to supervisord.log, which the teardown + log check cannot ignore. Re-enable this (and assert the #2151 governance-vs-docs + behaviour) once #2150 is fixed. + """ + pytest.skip("only_utxo preset crashes db-sync, see cardano-db-sync issue #2150") + + yield preset_only_utxo + + def preset_only_governance( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `only_governance` preset: governance data, no tx_out / multi_asset.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_GOVERNANCE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.NOT_EMPTY, + } + ) + + yield preset_only_governance + + def preset_disable_all( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `disable_all` preset: only block/tx and ledger-related data.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.DISABLE_ALL) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.EMPTY, + db_sync.Table.REDEEMER: TableCondition.EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.EMPTY, + } + ) + + yield preset_disable_all + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, cluster_singleton: clusterlib.ClusterLib, db_sync_manager: db_sync.DBSyncManager, subtests: pytest_subtests.SubTests, + worker_id: str, ): """Test DB-Sync configuration options using multiple subtests. Verifies that different DB-Sync configuration settings correctly control table population - and data insertion behavior. Each subtest modifies the configuration, restarts DB-Sync, - and validates the expected database state. - - * Test `tx_out` option (enable/disable modes with various settings) - * Verify address, tx_in, tx_out, and ma_tx_out tables respond to tx_out configuration - * Test `governance` option (enable/disable) - * Verify all governance-related tables populate when enabled and clear when disabled - * Test `tx_cbor` option (enable/disable) - * Verify tx_cbor table populates when enabled and clears when disabled - * Test `multi_asset` option (enable/disable) - * Verify multi_asset table populates when enabled and clears when disabled + and data insertion behavior. Each subtest modifies the configuration, restarts DB-Sync + (recreating and re-syncing the database), and validates the expected database state. + + Covers, grouped by config option: + + * `tx_out` (enable/disable/consumed modes, force_tx_in, use_address_table) + * `governance`, `tx_cbor`, `multi_asset` (enable/disable) + * `plutus`, `metadata` (+ keys filter), `shelley` (enable/disable side effects) + * `ledger` (enable/disable/ignore) and `pool_stat` + * `offchain_pool_data` and `offchain_vote_data` (need --allow-private-offchain-urls) + * `remove_jsonb_from_schema` (column-type effects) + * `disable_epoch` (the `epoch` rollup view) + * insert-option presets (`full`, `only_utxo`, `only_governance`, `disable_all`) * Restore original DB-Sync configuration after all subtests complete """ cluster = cluster_singleton common.get_test_id(cluster) + # `pool_stat` logs a benign "assume the pool exists and move on" warning + # (queryPoolHashId / insertPoolStats) while a pool is not yet in the active cache. + # Ignore it so it does not fail the test during the teardown log check. + for _glob in ("dbsync.stdout", "dbsync.stderr"): + logfiles.add_ignore_rule( + files_glob=_glob, + regex="queryPoolHashId", + ignore_file_id=worker_id, + ) + for subt in self.get_subtests(): with subtests.test(scenario=getattr(subt, "__name__", "")): subt(db_sync_manager) diff --git a/cardano_node_tests/utils/dbsync_queries.py b/cardano_node_tests/utils/dbsync_queries.py index 4e8cac15a..2879714d3 100644 --- a/cardano_node_tests/utils/dbsync_queries.py +++ b/cardano_node_tests/utils/dbsync_queries.py @@ -1211,6 +1211,18 @@ def query_view_names() -> list[str]: return view_names +def query_column_data_type(*, table: str, column: str) -> str | None: + """Query the SQL data type of a column, or `None` if the column does not exist.""" + query = ( + "SELECT data_type FROM information_schema.columns " + "WHERE table_name = %s AND column_name = %s;" + ) + + with execute(query=query, vars=(table, column)) as cur: + result = cur.fetchone() + return result[0] if result is not None else None + + def query_datum(*, datum_hash: str) -> tp.Generator[DatumDBRow]: """Query datum record in db-sync.""" query = "SELECT id, hash, tx_id, value, bytes FROM datum WHERE hash = %s;" diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index 29f673d01..b57797be3 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -117,6 +117,7 @@ class Tx(enum.StrEnum): class Redeemer(enum.StrEnum): SCRIPT_HASH = "redeemer.script_hash" + FEE = "redeemer.fee" class SettingState(enum.StrEnum): @@ -185,73 +186,22 @@ def __init__(self) -> None: "plutus": PlutusConfig(), "governance": SettingState.ENABLE, "offchain_pool_data": SettingState.ENABLE, + "offchain_vote_data": SettingState.DISABLE, "pool_stat": SettingState.ENABLE, "remove_jsonb_from_schema": SettingState.DISABLE, + # Optional key: emitted only when explicitly set, otherwise db-sync's own default + # is used (keeps the config for all other subtests unchanged). + "disable_epoch": None, } self._preset_applied = False + self._preset: Preset | None = None def with_preset(self, *, preset: Preset) -> tp.Self: + # Emit only the `preset` key and let db-sync expand it with its own preset + # definitions (see ``build``). The per-option `with_*` builders no-op once a + # preset is selected, so the individual config values are not used here. self._preset_applied = True - - if preset == Preset.FULL: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.ENABLE), - "ledger": LedgerMode.ENABLE, - "shelley": ShelleyConfig(enable=True), - "multi_asset": MultiAssetConfig(enable=True), - "metadata": MetadataConfig(enable=True), - "plutus": PlutusConfig(enable=True), - "governance": SettingState.ENABLE, - "offchain_pool_data": SettingState.ENABLE, - "pool_stat": SettingState.ENABLE, - } - ) - elif preset == Preset.ONLY_UTXO: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.BOOTSTRAP), - "ledger": LedgerMode.IGNORE, - "shelley": ShelleyConfig(enable=False), - "metadata": MetadataConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=True), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.DISABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.DISABLE, - } - ) - elif preset == Preset.ONLY_GOVERNANCE: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.DISABLE), - "ledger": LedgerMode.ENABLE, - "shelley": ShelleyConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=False), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.ENABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.ENABLE, - } - ) - elif preset == Preset.DISABLE_ALL: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.DISABLE), - "ledger": LedgerMode.DISABLE, - "shelley": ShelleyConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=False), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.DISABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.DISABLE, - } - ) - + self._preset = preset return self def with_tx_cbor(self, *, value: SettingState) -> tp.Self: @@ -307,6 +257,11 @@ def with_offchain_pool_data(self, *, value: SettingState) -> tp.Self: self._config["offchain_pool_data"] = value return self + def with_offchain_vote_data(self, *, value: SettingState) -> tp.Self: + if not self._preset_applied: + self._config["offchain_vote_data"] = value + return self + def with_pool_stat(self, *, value: SettingState) -> tp.Self: if not self._preset_applied: self._config["pool_stat"] = value @@ -317,7 +272,17 @@ def with_remove_jsonb_from_schema(self, *, value: SettingState) -> tp.Self: self._config["remove_jsonb_from_schema"] = value return self + def with_disable_epoch(self, *, value: bool) -> tp.Self: + if not self._preset_applied: + self._config["disable_epoch"] = value + return self + def build(self) -> dict[str, tp.Any]: + # When a preset is selected, emit only the `preset` key so db-sync expands it with + # its own preset definitions (individual keys would override the preset base). + if self._preset is not None: + return {"preset": self._preset.value} + tx_out = tp.cast(TxOutConfig, self._config["tx_out"]) shelley = tp.cast(ShelleyConfig, self._config["shelley"]) multi_asset = tp.cast(MultiAssetConfig, self._config["multi_asset"]) @@ -341,10 +306,12 @@ def build(self) -> dict[str, tp.Any]: "plutus": {"enable": plutus.enable}, "governance": self._enum_to_value(self._config["governance"]), "offchain_pool_data": self._enum_to_value(self._config["offchain_pool_data"]), + "offchain_vote_data": self._enum_to_value(self._config["offchain_vote_data"]), "pool_stat": self._enum_to_value(self._config["pool_stat"]), "remove_jsonb_from_schema": self._enum_to_value( self._config["remove_jsonb_from_schema"] ), + **self._optional("disable_epoch", self._config["disable_epoch"]), } return config diff --git a/cardano_node_tests/utils/dbsync_utils.py b/cardano_node_tests/utils/dbsync_utils.py index 283311ecf..21d229ff1 100644 --- a/cardano_node_tests/utils/dbsync_utils.py +++ b/cardano_node_tests/utils/dbsync_utils.py @@ -4,6 +4,7 @@ import functools import itertools import logging +import pathlib as pl import time import typing as tp @@ -1666,6 +1667,38 @@ def table_exists(*, table: str) -> bool: return table in table_names +def column_exists(*, table: str, column: str) -> bool: + """Check if a column exists in a database table.""" + return dbsync_queries.query_column_data_type(table=table, column=column) is not None + + +def column_data_type(*, table: str, column: str) -> str | None: + """Return the SQL data type of a column, or `None` if it does not exist.""" + return dbsync_queries.query_column_data_type(table=table, column=column) + + +def allow_private_offchain_urls_enabled() -> bool: + """Check whether the running db-sync uses ``--allow-private-offchain-urls``. + + This is a start-time CLI flag (gated by the ``DBSYNC_ALLOW_PRIVATE_OFFCHAIN_URLS`` env + var in the ``run-cardano-dbsync`` script), not an insert option, so it cannot be toggled + per test. Off-chain fetching of private / localhost metadata URLs only works when it is + set. Detected from the running process arguments, because the run script always contains + the conditional flag and so its text is not a reliable signal. + """ + proc_root = pl.Path("/proc") + for proc_dir in proc_root.iterdir(): + if not proc_dir.name.isdigit(): + continue + try: + cmdline = (proc_dir / "cmdline").read_bytes() + except OSError: + continue + if b"cardano-db-sync" in cmdline and b"--allow-private-offchain-urls" in cmdline: + return True + return False + + def check_epoch_state(*, epoch_no: int, txid: str, action_type: ActionTypes) -> None: """Check governance stats per epoch in dbsync.""" if not configuration.HAS_DBSYNC: diff --git a/runner/regression.sh b/runner/regression.sh index 4c2c8e4d5..2cc27ddf1 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -64,6 +64,11 @@ elif [ "$MARKEXPR" = "conway only" ]; then elif [ "$MARKEXPR" = "dbsync config" ]; then export CLUSTERS_COUNT=1 export MARKEXPR="(dbsync and smoke) or dbsync_config" + # Allow db-sync to fetch off-chain metadata from the cluster's private/localhost + # URLs so the off-chain config tests can exercise offchain_pool_data / + # offchain_vote_data (the flag only permits private URLs; the tests still enable + # the corresponding insert options themselves). + export DBSYNC_ALLOW_PRIVATE_OFFCHAIN_URLS=true fi if [ -n "${CLUSTERS_COUNT:-}" ]; then