Upgrades, bugfixes and improvements

.cspell.json

@@ -103,6 +103,8 @@
     "unintuitive",
     "unpruned",
     "upserting",
+    "UTXO",
+    "UTXOs",
     "verack",
     "xprivkey",
     "xpubkey"

.gitignore

@@ -14,3 +14,4 @@ package-lock.json
 data
 .env
 client/generated
+.scratch

charts/chaingraph/templates/agent.yaml

@@ -41,6 +41,8 @@ spec:
           value: {{ .Values.agent.synchronousCommit | quote }}
         - name: CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB
           value: {{ .Values.agent.blockBufferTargetSizeMb | quote }}
+        - name: CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE
+          value: {{ .Values.agent.repairBlocksBatchSize | quote }}
         - name: CHAINGRAPH_GENESIS_BLOCKS
           value: {{ .Values.agent.genesisBlocks | quote }}
         - name: CHAINGRAPH_TRUSTED_NODES

charts/chaingraph/values.yaml

@@ -18,12 +18,15 @@ agent:
   # The maximum number of connections the agent should maintain to the database.
   # For best performance, this should be set to the number of CPUs available to Postgres. If not set, Chaingraph will assume that Postgres is running on hardware equivalent to its own. (This is ideal if Postgres is running on either the same machine or an equivalent one from a homogenous Kubernetes node pool.)
   maxConnections: ''
-# If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync.
-# In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default.
+  # If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync.
+  # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default.
   synchronousCommit: true
   # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync.
   # For best performance, this should be around `maxConnections * maximum block size`, while leaving enough memory available to the host machine. If left unset (recommended), Chaingraph will measure free memory at startup and attempt to select a reasonable value.
   blockBufferTargetSizeMb: ''
+  # Self-healing for https://github.com/bitauth/chaingraph/issues/74: after initial sync, the agent scans saved blocks in batches to find blocks whose linked transactions don't match the saved block size, then repairs those blocks by re-requesting them from trusted nodes.
+  # Set to 0 to disable the startup repair task.
+  repairBlocksBatchSize: 10000
   # A mapping of network magic bytes to hex-encoded genesis blocks.
   # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated.
   # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex

defaults.env

@@ -16,6 +16,21 @@ CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT=true
 # For best performance, this should be around `CHAINGRAPH_POSTGRES_MAX_CONNECTIONS * maximum block size`, while leaving enough memory available to the host machine. If not set, Chaingraph will measure free memory at startup and attempt to select a reasonable value.
 CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB=
 
+# Self-healing for https://github.com/bitauth/chaingraph/issues/74: after
+# initial sync, the agent scans saved blocks in batches to find blocks whose
+# linked transactions don't match the saved block size, then repairs those
+# blocks by re-requesting them from trusted nodes. Set to 0 to disable the
+# startup repair task.
+CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000
+
+# BCHN expires transactions from the mempool after 336 hours but does not announce
+# those expirations. At initial sync and periodically thereafter the agent scans for
+# node_transaction rows expiring soon and schedules each row to be archived to
+# node_transaction_history at its exact expiry time. The existing mempool
+# invalidation cascade archives same-node descendants. See bitauth/chaingraph#73.
+CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS=1209600000
+CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS=3600000
+
 # A mapping of network magic bytes to hex-encoded genesis blocks.
 # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated.
 # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex

images/bitcoin-cash-node/Dockerfile

@@ -7,9 +7,9 @@ RUN set -ex \
   && apt-get install -qq --no-install-recommends ca-certificates gosu wget \
   && rm -rf /var/lib/apt/lists/*
 
-ENV BITCOIN_VERSION 28.0.2
-ENV BITCOIN_URL https://download.bitcoincashnode.org/misc/builds/upgrade12_temp/linux/bitcoin-cash-node-28.0.2-x86_64-linux-gnu.tar.gz
-ENV BITCOIN_SHA256 140b44fd76a4f9428354bfbec4800d58fd39fb723320e761a035f15c2dd43596
+ENV BITCOIN_VERSION 29.0.0
+ENV BITCOIN_URL https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v29.0.0/bitcoin-cash-node-29.0.0-x86_64-linux-gnu.tar.gz
+ENV BITCOIN_SHA256 6125d1cbecc1db476f2b6b7b91da5acde92d2311b8e738124e3db64ca84b33e1
 
 # install bitcoin binaries
 RUN set -ex \

images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql

@@ -0,0 +1,2 @@
+DROP TRIGGER IF EXISTS trigger_public_node_transaction_history_insert ON node_transaction_history;
+DROP FUNCTION IF EXISTS trigger_node_transaction_history_insert();

images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql

@@ -0,0 +1,105 @@
+CREATE OR REPLACE FUNCTION trigger_node_transaction_history_insert() RETURNS trigger
+  LANGUAGE plpgsql
+AS $$
+BEGIN
+  /*
+   * The recursive CTE archives the full descendant set in one pass. The
+   * archive INSERT below re-fires this trigger, but that re-entry should return
+   * immediately rather than attempting another cascade.
+   */
+  IF current_setting('chaingraph.suppress_mempool_descendant_cascade', true) = 'on' THEN
+    RETURN NULL;
+  END IF;
+
+  -- Confirmations and empty batches do not invalidate descendants.
+  IF NOT EXISTS (SELECT 1 FROM new_table WHERE replaced_at IS NOT NULL) THEN
+    RETURN NULL;
+  END IF;
+
+  PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true);
+
+  BEGIN
+    /*
+     * If another session deletes matching descendants before this DELETE
+     * reaches them, this archives zero rows. Suppression still prevents empty
+     * self-reentry.
+     */
+    WITH RECURSIVE descendant_transactions AS (
+        -- Seed: mempool transactions spending outputs of newly replaced parents.
+        SELECT nt.node_internal_id,
+               nt.transaction_internal_id,
+               nt.validated_at,
+               replaced_parents.replaced_at
+            FROM new_table replaced_parents
+            INNER JOIN transaction parent_transaction
+                ON parent_transaction.internal_id = replaced_parents.transaction_internal_id
+            INNER JOIN output parent_output
+                ON parent_output.transaction_hash = parent_transaction.hash
+            INNER JOIN input
+                ON input.outpoint_transaction_hash = parent_output.transaction_hash
+               AND input.outpoint_index            = parent_output.output_index
+            INNER JOIN node_transaction nt
+                ON nt.transaction_internal_id = input.transaction_internal_id
+               AND nt.node_internal_id        = replaced_parents.node_internal_id
+            WHERE replaced_parents.replaced_at IS NOT NULL
+
+        UNION
+
+        -- Recursive step: mempool transactions spending outputs of descendants.
+        SELECT child_nt.node_internal_id,
+               child_nt.transaction_internal_id,
+               child_nt.validated_at,
+               parent_descendants.replaced_at
+            FROM descendant_transactions parent_descendants
+            INNER JOIN transaction parent_transaction
+                ON parent_transaction.internal_id = parent_descendants.transaction_internal_id
+            INNER JOIN output parent_output
+                ON parent_output.transaction_hash = parent_transaction.hash
+            INNER JOIN input
+                ON input.outpoint_transaction_hash = parent_output.transaction_hash
+               AND input.outpoint_index            = parent_output.output_index
+            INNER JOIN node_transaction child_nt
+                ON child_nt.transaction_internal_id = input.transaction_internal_id
+               AND child_nt.node_internal_id        = parent_descendants.node_internal_id
+    ),
+    descendants AS (
+        -- If reachable through multiple replaced parents, use earliest invalidation.
+        SELECT node_internal_id,
+               transaction_internal_id,
+               validated_at,
+               MIN(replaced_at) AS replaced_at
+            FROM descendant_transactions
+            GROUP BY node_internal_id, transaction_internal_id, validated_at
+    ),
+    deleted_descendants AS (
+        DELETE FROM node_transaction
+            USING descendants
+            WHERE node_transaction.node_internal_id        = descendants.node_internal_id
+              AND node_transaction.transaction_internal_id = descendants.transaction_internal_id
+            RETURNING node_transaction.node_internal_id,
+                      node_transaction.transaction_internal_id,
+                      node_transaction.validated_at,
+                      descendants.replaced_at
+    )
+    INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at)
+        SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at
+            FROM deleted_descendants;
+  EXCEPTION WHEN OTHERS THEN
+    PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true);
+    RAISE;
+  END;
+
+  PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true);
+  RETURN NULL;
+END;
+$$;
+
+CREATE TRIGGER trigger_public_node_transaction_history_insert
+    AFTER INSERT ON node_transaction_history
+    REFERENCING NEW TABLE AS new_table
+    FOR EACH STATEMENT EXECUTE FUNCTION trigger_node_transaction_history_insert();
+COMMENT ON TRIGGER trigger_public_node_transaction_history_insert ON node_transaction_history
+  IS 'Cascades mempool invalidation recursively: when a node_transaction is archived to history with replaced_at set, all same-node descendants still present in node_transaction are archived with a deterministic replaced_at timestamp.';
+
+-- disabled until initial sync is complete (when mempool transactions begin to be accepted)
+ALTER TABLE node_transaction_history DISABLE TRIGGER trigger_public_node_transaction_history_insert;

images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql

@@ -0,0 +1,2 @@
+-- This migration archives orphaned mempool descendants into
+-- node_transaction_history. The data move is intentionally not reversible.

images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql

@@ -0,0 +1,88 @@
+DO $$
+DECLARE
+  previous_suppression text := current_setting('chaingraph.suppress_mempool_descendant_cascade', true);
+BEGIN
+  /*
+   * The backfill archives the full existing orphan set itself. If the
+   * node_transaction_history trigger is enabled while this migration runs,
+   * suppress its re-entry from the archive INSERT below.
+   */
+  PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true);
+
+  BEGIN
+    WITH RECURSIVE orphan_transactions AS (
+        -- Seed: mempool transactions whose parent was already archived as replaced.
+        SELECT nt.node_internal_id,
+               nt.transaction_internal_id,
+               nt.validated_at,
+               nth.replaced_at
+            FROM node_transaction nt
+            INNER JOIN input child_input
+                ON child_input.transaction_internal_id = nt.transaction_internal_id
+            INNER JOIN transaction parent_transaction
+                ON parent_transaction.hash = child_input.outpoint_transaction_hash
+            INNER JOIN output parent_output
+                ON parent_output.transaction_hash = parent_transaction.hash
+               AND parent_output.output_index      = child_input.outpoint_index
+            INNER JOIN node_transaction_history nth
+                ON nth.transaction_internal_id = parent_transaction.internal_id
+               AND nth.node_internal_id        = nt.node_internal_id
+            WHERE nth.replaced_at IS NOT NULL
+
+        UNION
+
+        -- Recursive step: mempool transactions spending outputs of known orphans.
+        SELECT child_nt.node_internal_id,
+               child_nt.transaction_internal_id,
+               child_nt.validated_at,
+               parent_orphans.replaced_at
+            FROM orphan_transactions parent_orphans
+            INNER JOIN transaction parent_transaction
+                ON parent_transaction.internal_id = parent_orphans.transaction_internal_id
+            INNER JOIN output parent_output
+                ON parent_output.transaction_hash = parent_transaction.hash
+            INNER JOIN input child_input
+                ON child_input.outpoint_transaction_hash = parent_output.transaction_hash
+               AND child_input.outpoint_index            = parent_output.output_index
+            INNER JOIN node_transaction child_nt
+                ON child_nt.transaction_internal_id = child_input.transaction_internal_id
+               AND child_nt.node_internal_id        = parent_orphans.node_internal_id
+    ),
+    orphans AS (
+        -- If reachable through multiple replaced parents, use earliest invalidation.
+        SELECT node_internal_id,
+               transaction_internal_id,
+               validated_at,
+               MIN(replaced_at) AS replaced_at
+            FROM orphan_transactions
+            GROUP BY node_internal_id, transaction_internal_id, validated_at
+    ),
+    deleted_orphans AS (
+        DELETE FROM node_transaction
+            USING orphans
+            WHERE node_transaction.node_internal_id        = orphans.node_internal_id
+              AND node_transaction.transaction_internal_id = orphans.transaction_internal_id
+            RETURNING node_transaction.node_internal_id,
+                      node_transaction.transaction_internal_id,
+                      node_transaction.validated_at,
+                      orphans.replaced_at
+    )
+    INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at)
+        SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at
+            FROM deleted_orphans;
+  EXCEPTION WHEN OTHERS THEN
+    PERFORM set_config(
+      'chaingraph.suppress_mempool_descendant_cascade',
+      COALESCE(previous_suppression, 'off'),
+      true
+    );
+    RAISE;
+  END;
+
+  PERFORM set_config(
+    'chaingraph.suppress_mempool_descendant_cascade',
+    COALESCE(previous_suppression, 'off'),
+    true
+  );
+END;
+$$;

images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql

@@ -0,0 +1,25 @@
+CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea
+  LANGUAGE plpgsql IMMUTABLE
+AS $$
+DECLARE
+  transactions CURSOR FOR SELECT transaction.* FROM transaction
+    INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id
+    WHERE block_transaction.block_internal_id = block_row.internal_id
+    ORDER BY block_transaction.transaction_index ASC;
+  encoded_block bytea := encode_block_header(block_row) || encode_compact_uint(COUNT(transactions));
+BEGIN
+  FOR transaction_row IN transactions
+  LOOP
+    encoded_block := encoded_block ||
+      encode_transaction(ROW(
+        transaction_row.internal_id,
+        transaction_row.hash,
+        transaction_row.version,
+        transaction_row.locktime,
+        transaction_row.size_bytes,
+        transaction_row.is_coinbase)::transaction
+      );
+  END LOOP;
+  RETURN encoded_block;
+END;
+$$;

images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql

@@ -0,0 +1,30 @@
+CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea
+  LANGUAGE plpgsql IMMUTABLE
+AS $$
+DECLARE
+  transactions CURSOR FOR SELECT transaction.* FROM transaction
+    INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id
+    WHERE block_transaction.block_internal_id = block_row.internal_id
+    ORDER BY block_transaction.transaction_index ASC;
+  encoded_block bytea := encode_block_header(block_row);
+  transaction_count bigint := 0;
+BEGIN
+  SELECT COUNT(*) INTO transaction_count
+    FROM block_transaction
+    WHERE block_transaction.block_internal_id = block_row.internal_id;
+  encoded_block := encoded_block || encode_compact_uint(transaction_count);
+  FOR transaction_row IN transactions
+  LOOP
+    encoded_block := encoded_block ||
+      encode_transaction(ROW(
+        transaction_row.internal_id,
+        transaction_row.hash,
+        transaction_row.version,
+        transaction_row.locktime,
+        transaction_row.size_bytes,
+        transaction_row.is_coinbase)::transaction
+      );
+  END LOOP;
+  RETURN encoded_block;
+END;
+$$;

images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql

@@ -0,0 +1,5 @@
+CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output
+  LANGUAGE sql IMMUTABLE
+AS $$
+  SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR get_byte(locking_bytecode, 0) = 106);
+$$;

images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql

@@ -0,0 +1,5 @@
+CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output
+  LANGUAGE sql IMMUTABLE
+AS $$
+  SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR (octet_length(locking_bytecode) > 0 AND get_byte(locking_bytecode, 0) = 106));
+$$;