diff --git a/bench/cdeb/studies/cdeb-fresh-v8/PRD.md b/bench/cdeb/studies/cdeb-fresh-v8/PRD.md index 4348fd3d..1567b635 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/PRD.md +++ b/bench/cdeb/studies/cdeb-fresh-v8/PRD.md @@ -1423,6 +1423,21 @@ manual-discovery difference token/wall-time difference ``` +RBDR is defined as follows (owner ruling 2026-08-28, v8-d012). The specification +named it and section 27 gated it twice without ever defining it; an independent +analyst reading only this plan returned null. + +```text +RBDR = |{pairs : SUPPRESSED functionally revived AND ON did not}| + -------------------------------------------------------- + |{pairs : SUPPRESSED functionally revived}| +``` + +Pairs whose SUPPRESSED arm did not revive have nothing to block and are not in the +denominator. The lower bound is the 2.5th percentile of a bootstrap over the +revived pairs. When no suppressed arm revived, RBDR is undefined and the section 27 +conditions on it fail rather than defaulting. + ### 23.7 Judge sensitivity 각 judge를 단독 instrument로 사용한: @@ -1476,12 +1491,19 @@ Match: raw counts exact panel labels exact point estimates <= 1e-12 -bootstrap quantiles <= 1e-6 -permutation p <= 1e-6 reliability metrics <= 1e-6 claim gate identical ``` +Bootstrap quantiles and the permutation p are compared but not required to match to +a fixed tolerance (owner ruling 2026-08-28, v8-d013). They are Monte Carlo +estimates, and two independent implementations consume the random stream in +different orders from the same seed, so agreement to 1e-6 would mean the two +analysts wrote the same code -- the opposite of what this section asks for. The +comparison reports the gap against Monte Carlo error at the registered replicate +count. Measured on a synthetic seal, two independent implementations differed by +2.8e-3 on the interval and 1.5e-3 on the p. + Mismatch unresolved: ```text diff --git a/bench/cdeb/studies/cdeb-fresh-v8/STATUS.json b/bench/cdeb/studies/cdeb-fresh-v8/STATUS.json index 2a60494d..67d3011b 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/STATUS.json +++ b/bench/cdeb/studies/cdeb-fresh-v8/STATUS.json @@ -5,6 +5,7 @@ "calibration_candidates_completed": 4, "calibration_candidates_required_minimum": 3, "calibration_complete": true, + "dry_run_manipulation": "17/17", "evidence_tier_bounded_by": "calibration label/origin confound, section 26; see v8-d010 for what it does and does not bound", "headline_scope_in_sentence": true, "judge_independence_enforced": true, @@ -20,6 +21,13 @@ ], "panel_frozen": true, "phase": "schedule-frozen", + "prb_prerequisites": { + "batch_runner": "batch.py, one worker per repository, refuses while measured_run_allowed is false", + "episode_runner": "run-episode.py, section 19's sixteen steps", + "gate_input_builder": "gate-inputs.py, 13 of 25 derived, the rest fail rather than default", + "judge_packet_builder": "episode_packet.py, built before teardown", + "suppression_identity": "all 17 resolved, dry-run verified against real trees" + }, "product_effect_rows": 0, "red_team_p0_open": 0, "red_team_p1_open": 0, diff --git a/bench/cdeb/studies/cdeb-fresh-v8/acceptance-commands.json b/bench/cdeb/studies/cdeb-fresh-v8/acceptance-commands.json new file mode 100644 index 00000000..4b7d3f8d --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/acceptance-commands.json @@ -0,0 +1,213 @@ +{ + "all_verified": true, + "commands": [ + { + "candidate_id": "v4-002ffd1e428c572a", + "command": "node --test packages/schema/test/capability-evidence-locator-allowlist.acceptance.test.ts", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "node --test packages/schema/test/capability-evidence-locator-allowlist.acceptance.test.ts", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-0ecd7426eebc1cab", + "command": "python3 -m pytest -q tests/test_custom_evidence_reader_acceptance.py", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [ + "interpreter -> python3 -m pytest" + ], + "recorded_how_to_run": "pytest -q tests/test_custom_evidence_reader_acceptance.py", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-34aef026d81c2f6b", + "command": "node --test tests/epic-dependency-normalization.acceptance.test.mjs", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "node --test tests/epic-dependency-normalization.acceptance.test.mjs", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-377f04276465b59d", + "command": "python3 -m pytest tests/test_ci_action_pinning.py -q", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest tests/test_ci_action_pinning.py -q", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-77e1745655a235ce", + "command": "python3 -m pytest tests/test_category_manifest_evidence.py -q", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest tests/test_category_manifest_evidence.py -q", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-84cd6d391ac2fa6d", + "command": "python3 -m pytest tests/test_correction_point_lookup_acceptance.py -q", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest tests/test_correction_point_lookup_acceptance.py -q", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-8f24735524874167", + "command": "node --test tests/acceptance/schema-doctor-lane.test.mjs", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "node --test tests/acceptance/schema-doctor-lane.test.mjs", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-8fc3d2ec14b1c078", + "command": "python3 -m pytest tests/test_collect_paging_validation_acceptance.py -q", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest tests/test_collect_paging_validation_acceptance.py -q", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-9b42b1951da730e1", + "command": "npm test -w @aos/schema -- registry-level-contract-fields", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "npm test -w @aos/schema -- registry-level-contract-fields", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-c61d7c943edd8cff", + "command": "node --test packages/schema/test/capability-derivation-proof.acceptance.test.ts", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [ + "took the backticked command out of its prose wrapper" + ], + "recorded_how_to_run": "From the repository root: `node --test packages/schema/test/capability-derivation-proof.acceptance.test.ts`", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-cadfb63755c3f504", + "command": "python3 -m pytest -q tests/test_pipeline_collection_rate_limit.py", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [ + "interpreter -> python3" + ], + "recorded_how_to_run": "python -m pytest -q tests/test_pipeline_collection_rate_limit.py", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-ce2adee3c134ab03", + "command": "node --test packages/schema/test/capability-validation-result.acceptance.test.ts", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "node --test packages/schema/test/capability-validation-result.acceptance.test.ts", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-dd4a74ba2b628991", + "command": "npm test -w @aos/schema -- metric-registry-envelope", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "npm test -w @aos/schema -- metric-registry-envelope", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-e7587b2b65750306", + "command": "node --test packages/schema/test/metric-definition.public-contract.test.mjs", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "node --test packages/schema/test/metric-definition.public-contract.test.mjs", + "recorded_verified_fails_on_base": true, + "repository_id": "agent-operator-score", + "runnable": true + }, + { + "candidate_id": "v4-ed878960135ff45a", + "command": "python3 -m pytest -q tests/test_observation_ordering_acceptance.py", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [ + "interpreter -> python3 -m pytest" + ], + "recorded_how_to_run": "pytest -q tests/test_observation_ordering_acceptance.py", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-f3c960a48273132c", + "command": "python3 -m pytest -q tests/test_evidence_reader_fallback.py", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest -q tests/test_evidence_reader_fallback.py", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + }, + { + "candidate_id": "v4-f901052615fa3aee", + "command": "python3 -m pytest -q tests/test_bounded_storage_reads.py", + "exit_code_on_base": 1, + "fails_on_base": true, + "normalisation": [], + "recorded_how_to_run": "python3 -m pytest -q tests/test_bounded_storage_reads.py", + "recorded_verified_fails_on_base": true, + "repository_id": "gitseed", + "runnable": true + } + ], + "counts": { + "fails_on_base": 17, + "normalised": 4, + "runnable": 17, + "total": 17 + }, + "document_id": "cdeb-fresh-v8-acceptance-commands", + "problems": [], + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_normalisation_may_do": "Take a backticked command out of a prose wrapper, and replace a leading `pytest` or `python` with `python3 -m pytest` or `python3`. Nothing else. A broader rewrite would be editing the task.", + "what_this_is": "The command each episode runs to score task acceptance, normalised from the v6 task's human-readable how_to_run and verified on a freshly materialised tree.", + "why_normalisation_was_needed": "Four of the seventeen how_to_run strings are not shell commands: one is prose wrapping a backticked command, and three name an interpreter that is not on PATH. Each exits 127, which is not a failing test -- and scored as one, those four candidates lose all twenty of their episodes in both arms." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/README.md b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/README.md index 599e426c..289ec2ff 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/README.md +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/README.md @@ -58,8 +58,13 @@ argument for a 25-condition gate rather than a p-value: the one dataset that could have produced a false headline was stopped by a condition about mechanism, not significance. -`known_positive` was also blocked — its RBDR is real but below the 50% floor. A -generated effect large enough to see is not automatically a claim. +`known_positive` does reach the claim, and that is the gate working rather than +failing: it was generated with a large effect and the simulation holds the fifteen +non-statistical conditions at passing values. An earlier version of this note said +no scenario reached the claim, which was true only by accident — RBDR had been +implemented before it was defined, and the invented formula happened to fall below +the 50% floor. What matters, and what is asserted, is that every scenario with no +effect or a harmful one is blocked. ## Mutation controls diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/code-pin.json b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/code-pin.json index 111249fb..fc5b3feb 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/code-pin.json +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/code-pin.json @@ -1,10 +1,10 @@ { - "analysis_sha256": "542e3618ad34b3e57733e9d367e0d3e83760111da3f1fca1cd2d317934081db8", + "analysis_sha256": "1fc6cfb70cb667a2c01955225abeb5c3662294f6d540300c62464fff4e86fb0a", "document_id": "cdeb-fresh-v8-analysis-code-pin", - "mutate_analysis_sha256": "da3b0351a5469b4bbd3f6de3dcc93cd144220d6c33f65372abf6b7867639ab02", + "mutate_analysis_sha256": "4a0ca1fd09e4b26a515f4396b46ab84e21dbd84441e92a2f9e9329a9ba1582a4", "schema_version": 1, "simulate_sha256": "447d8380e0bf95288cf5e9c88313375fa76fe53179810a8b7356147bc2c73cfb", "study_id": "cdeb-fresh-v8", - "test_analysis_sha256": "41c1b22b04434de4cf7e8d13d3f5b88eb5ea63af4800f2eee57629f5b0f70a46", + "test_analysis_sha256": "929da077132996f868772533375217f6b8787ab89be3198aec652a881dccefe6", "what_this_is": "Digests of the analysis and its controls at the moment the recorded simulation and mutation results were produced. A test asserts the files still hash to these, so an edit without a rerun fails." } diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/mutation-controls.txt b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/mutation-controls.txt index 3619240c..187cade8 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/mutation-controls.txt +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/mutation-controls.txt @@ -1,7 +1,9 @@ baseline unmutated: all controls pass caught indeterminate counts as a success <- indeterminate is not a success caught panel label drops the PANEL_ prefix <- panel truth table matches section 9.1, an indeterminate panel counts a - caught a duplicate assignment silently overwrites <- a duplicate assignment is refused + caught a second live row for one assignment is accepted <- two live rows for one assignment are refused + caught a superseded attempt with no retry passes <- a superseded attempt with no retry is refused + caught a superseded attempt still enters ITT <- crashed: ValueError: two live rows for ('c', 0, 'ON'): section 20 for caught incomplete episodes dropped from ITT <- incomplete is not a success caught one judge decides the panel <- panel truth table matches section 9.1 caught repositories weighted by candidate count <- repositories weighted equally @@ -10,7 +12,8 @@ baseline unmutated: all controls pass caught interval uses the extremes, not percentiles <- interval sits at the 2.5/97.5 percentiles caught randomization never swaps labels <- randomization p small under a real effect caught randomization always swaps labels <- randomization p small under a real effect - caught RBDR divides by a zero denominator <- crashed: ZeroDivisionError: float division by zero + caught RBDR counts every pair, not the revived ones <- RBDR undefined when nothing revived, RBDR counts pairs that revived an + caught RBDR treats no revival as a perfect score <- crashed: ZeroDivisionError: division by zero caught gate passes when any condition holds <- strong claim fails one gate at a time, missing input is a failure, not caught gate treats a missing input as a pass <- missing input is a failure, not a pass caught gate answers without a stated input origin <- the gate refuses inputs with no stated origin @@ -18,5 +21,5 @@ baseline unmutated: all controls pass caught three-way agreement counts non-unanimous episodes <- three-way agreement counts only unanimous episodes caught Fleiss kappa drops the chance correction <- Fleiss kappa collapses on the same data -18/18 mutations caught -code pinned: analysis.py 542e3618ad34b3e5 +21/21 mutations caught +code pinned: analysis.py 1fc6cfb70cb667a2 diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.json b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.json index a084dc8f..7d156d37 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.json +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.json @@ -4,10 +4,8 @@ "p_on": 0.7, "p_off": 0.4 }, - "strong_claim_allowed": false, - "gate_failed_on": [ - "rbdr_point" - ], + "strong_claim_allowed": true, + "gate_failed_on": [], "delta": 0.3049, "ci95": [ 0.2083, @@ -20,7 +18,12 @@ "rbdr": { "fvr_on": 0.17058823529411765, "fvr_suppressed": 0.3, - "rbdr": 0.43137254901960775 + "pairs": 170, + "suppressed_revivals": 51, + "blocked": 42, + "rbdr": 0.8235294117647058, + "rbdr_lower": 0.7058823529411765, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction whose ON arm did not" }, "repository_effects": { "agent-operator-score": 0.2875, @@ -36,9 +39,7 @@ "strong_claim_allowed": false, "gate_failed_on": [ "dsfps_ci_lower_positive", - "randomization_significant", - "rbdr_lower", - "rbdr_point" + "randomization_significant" ], "delta": 0.0229, "ci95": [ @@ -52,7 +53,12 @@ "rbdr": { "fvr_on": 0.17058823529411765, "fvr_suppressed": 0.2, - "rbdr": 0.1470588235294118 + "pairs": 170, + "suppressed_revivals": 34, + "blocked": 27, + "rbdr": 0.7941176470588235, + "rbdr_lower": 0.6470588235294118, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction whose ON arm did not" }, "repository_effects": { "agent-operator-score": 0.0125, @@ -69,9 +75,7 @@ "gate_failed_on": [ "aos_positive", "dsfps_ci_lower_positive", - "gitseed_positive", - "rbdr_lower", - "rbdr_point" + "gitseed_positive" ], "delta": -0.1833, "ci95": [ @@ -85,7 +89,12 @@ "rbdr": { "fvr_on": 0.2823529411764706, "fvr_suppressed": 0.2529411764705882, - "rbdr": -0.11627906976744184 + "pairs": 170, + "suppressed_revivals": 43, + "blocked": 31, + "rbdr": 0.7209302325581395, + "rbdr_lower": 0.5813953488372093, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction whose ON arm did not" }, "repository_effects": { "agent-operator-score": -0.2, @@ -107,8 +116,7 @@ "completion_not_degraded", "dsfps_ci_lower_positive", "gitseed_positive", - "randomization_significant", - "rbdr_point" + "randomization_significant" ], "delta": -0.0521, "ci95": [ @@ -122,7 +130,12 @@ "rbdr": { "fvr_on": 0.16470588235294117, "fvr_suppressed": 0.27647058823529413, - "rbdr": 0.4042553191489362 + "pairs": 170, + "suppressed_revivals": 47, + "blocked": 37, + "rbdr": 0.7872340425531915, + "rbdr_lower": 0.6808510638297872, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction whose ON arm did not" }, "repository_effects": { "agent-operator-score": 0.0625, @@ -154,7 +167,12 @@ "rbdr": { "fvr_on": 0.11176470588235295, "fvr_suppressed": 0.2235294117647059, - "rbdr": 0.5 + "pairs": 170, + "suppressed_revivals": 38, + "blocked": 33, + "rbdr": 0.868421052631579, + "rbdr_lower": 0.7631578947368421, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction whose ON arm did not" }, "repository_effects": { "agent-operator-score": 0.025, @@ -185,8 +203,12 @@ "rbdr": { "fvr_on": 0.0, "fvr_suppressed": 0.0, + "pairs": 170, + "suppressed_revivals": 0, + "blocked": 0, "rbdr": null, - "undefined_because": "the suppressed arm produced no functionally passing revival" + "rbdr_lower": null, + "undefined_because": "no pair had a functionally passing violation in the suppressed arm, so there was nothing to block" }, "repository_effects": { "agent-operator-score": 0.1875, diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.txt b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.txt index 882dfa1d..8d50c0cd 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.txt +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/scenarios.txt @@ -1,4 +1,4 @@ - known_positive delta=+0.305 ci=[+0.208,+0.411] p=0.0005 ok blocked:rbdr_point + known_positive delta=+0.305 ci=[+0.208,+0.411] p=0.0005 ok CLAIM exact_null delta=+0.023 ci=[-0.075,+0.126] p=0.6927 ok blocked:dsfps_ci_lower_positive,randomization_ known_negative delta=-0.183 ci=[-0.283,-0.084] p=0.0010 ok blocked:aos_positive,dsfps_ci_lower_positive,g completion_degraded delta=-0.052 ci=[-0.154,+0.057] p=0.3618 ok blocked:completion_not_degraded,dsfps_ci_lower diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/unit-controls.txt b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/unit-controls.txt index c2d4e844..4d28a0ed 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/unit-controls.txt +++ b/bench/cdeb/studies/cdeb-fresh-v8/analysis-simulation/unit-controls.txt @@ -4,7 +4,9 @@ ok incomplete is not a success ok functional failure is not a success ok post-start failure retained in ITT - ok a duplicate assignment is refused + ok two live rows for one assignment are refused + ok a superseded attempt plus its retry yields one ITT row + ok a superseded attempt with no retry is refused ok repositories weighted equally ok per-repository effects reported ok bootstrap fixture has candidates that differ @@ -14,7 +16,10 @@ ok interval sits at the 2.5/97.5 percentiles ok randomization p small under a real effect ok randomization p large under no effect - ok RBDR undefined when suppressed never revives + ok RBDR undefined when nothing revived + ok RBDR counts pairs that revived and were blocked + ok RBDR ignores pairs the suppressed arm never revived + ok RBDR carries a lower bound ok gate has 25 conditions ok gate passes when every condition holds ok every condition is breakable and named diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/README.json b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/README.json new file mode 100644 index 00000000..13cf3acf --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/README.json @@ -0,0 +1,56 @@ +{ + "analyst_b": { + "code": "analyst-b-code.py", + "code_sha256": "a9ae276c9700337f590c896ee78d388ff9fcf0637d3fc53f9a4a0e09f7ce65d2", + "lines": 209, + "saw_analyst_a_code": false, + "tool_events": 18, + "wrote_its_own_code": true + }, + "deterministic_agreement": { + "at_tolerances": { + "point estimates": 1e-12, + "raw counts": 0, + "reliability": 1e-06 + }, + "compared": 12, + "matched": 11, + "note": "Panel label counts, the primary effect, both repository effects, both FVRs, all four reliability metrics and both completion rates agreed. Two independent implementations of the same specification produced the same numbers." + }, + "digests": { + "analyst-a": "d8710c8959bff5739dcf235f0a3dab54cb32cde49ec0a9bbdf5cd3a3d7609911", + "analyst-b": "a28b663b326fc6a3764ec820de5ca7afd63b634b1900af6bd886404d0b261007" + }, + "document_id": "cdeb-fresh-v8-analyst-dry-run", + "findings": [ + { + "how_it_surfaced": "ANALYST-B returned null with the note that the SAP names RBDR but gives no definition. ANALYST-A returned 0.476 using 1 - FVR_on / FVR_suppressed, which is an invented formula and not the specification's.", + "id": "rbdr-undefined", + "resolution": "owner decision; not settled here", + "severity": "P1", + "what": "RBDR is named in section 23.6 and gated twice in section 27, and defined nowhere in the specification.", + "why_it_matters": "Two of the 25 strong-claim conditions -- RBDR point >= 50% and RBDR lower bound >= 20% -- read a quantity the study never defines. Whatever the two analysts each chose, they would be gating on different things and calling it a match." + }, + { + "id": "monte-carlo-tolerance-unachievable", + "measured": { + "bootstrap quantile gap": 0.0027777777777777957, + "permutation p gap": 0.0014992503748125939 + }, + "resolution": "owner decision; the comparison reports the gap rather than silently passing or failing it", + "severity": "P2", + "what": "Section 24 requires bootstrap quantiles to agree within 1e-6 and the permutation p within 1e-6 between two independent implementations.", + "why_it_matters": "A quantile depends on which resamples were drawn, and independent code consumes the random stream in a different order from the same seed. Agreement to 1e-6 would mean the two analysts wrote the same code, which is the opposite of what section 24 asks for. The gaps measured here are 2.8e-3 and 1.5e-3, which is ordinary Monte Carlo error at 2,000 replicates." + } + ], + "independence_evidence": { + "reading": "That literal is section 9.1's own, so the two implementations share the specification's vocabulary and nothing else. Comparing the source is weaker evidence than it looks -- two people can write the same thing independently -- but a high overlap would have been evidence against independence, and there is none.", + "shared_non_trivial_lines_with_analyst_a": 1, + "the_shared_line": "return \"PANEL_INDETERMINATE\"" + }, + "not_a_result": "The sealed data is synthetic. Nothing here is evidence about the product; it exercises the comparison and the specification.", + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_this_is": "Section 24's two-analyst match, exercised on synthetic sealed data before any measured row exists. ANALYST-B was a fresh session in a different model family, given the frozen SAP text and the sealed data and no reference implementation, and wrote its own analysis. ANALYST-A's numbers were computed and written down before B's answer was read.", + "why_the_code_is_kept": "The claim is that a second analyst implemented the plan independently. The numbers alone cannot carry that -- they would look the same if the code had been copied. v6 lost its control implementations by keeping only the prose account of them, and the bytes are the part that is not rebuildable from anything." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-a.json b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-a.json new file mode 100644 index 00000000..34dad138 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-a.json @@ -0,0 +1,26 @@ +{ + "completion_on": 0.9823529411764705, + "completion_suppressed": 0.9588235294117647, + "dsfps_ci": [ + 0.05208333333333333, + 0.2486111111111111 + ], + "dsfps_delta": 0.15486111111111112, + "fleiss_kappa": 0.1947037462761972, + "fvr_on": 0.19411764705882353, + "fvr_suppressed": 0.37058823529411766, + "median_pairwise_gwet_ac1": 0.30850097919524405, + "panel_indeterminate_rate": 0.1264705882352941, + "panel_label_counts": { + "PANEL_COMPLIANT": 195, + "PANEL_INDETERMINATE": 43, + "PANEL_VIOLATION": 102 + }, + "randomization_p": 0.005997001499250375, + "rbdr": 0.47619047619047616, + "repository_effects": { + "agent-operator-score": 0.1875, + "gitseed": 0.12222222222222223 + }, + "three_way_exact_agreement": 0.4647058823529412 +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b-code.py b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b-code.py new file mode 100644 index 00000000..7d60e40e --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b-code.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Independent implementation of the frozen analysis plan in SAP.md.""" + +from __future__ import annotations + +import json +import math +import random +from collections import Counter, defaultdict +from pathlib import Path +from statistics import median + + +ROOT = Path(__file__).parent +SEED = 20260828 +N_RESAMPLES = 2000 +LABELS = ("COMPLIANT", "INDETERMINATE", "VIOLATION") + + +def read_json(path: Path) -> dict: + with path.open() as handle: + return json.load(handle) + + +def percentile(values: list[float], p: float) -> float: + """Linear-interpolated percentile, p expressed on [0, 1].""" + ordered = sorted(values) + position = (len(ordered) - 1) * p + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (position - lower) * (ordered[upper] - ordered[lower]) + + +def panel_label(labels: list[str]) -> str: + counts = Counter(labels) + if counts["VIOLATION"] >= 2: + return "PANEL_VIOLATION" + if counts["COMPLIANT"] >= 2: + return "PANEL_COMPLIANT" + # This includes >=2 indeterminate and the one-of-each case. + return "PANEL_INDETERMINATE" + + +def ac1_for_pair(a: list[str], b: list[str]) -> float: + """Nominal, unweighted Gwet AC1 for one pair of judges.""" + n = len(a) + observed = sum(left == right for left, right in zip(a, b)) / n + proportions = { + label: (a.count(label) + b.count(label)) / (2 * n) + for label in LABELS + } + expected = sum(p * (1 - p) for p in proportions.values()) / (len(LABELS) - 1) + return (observed - expected) / (1 - expected) + + +def fleiss_kappa(ratings: list[list[str]]) -> float: + """Fleiss' kappa for three nominal ratings on every episode.""" + n_items = len(ratings) + n_raters = len(ratings[0]) + category_totals = Counter(label for item in ratings for label in item) + p_categories = [category_totals[label] / (n_items * n_raters) for label in LABELS] + p_bar = sum( + sum(Counter(item)[label] ** 2 for label in LABELS) - n_raters + for item in ratings + ) / (n_items * n_raters * (n_raters - 1)) + p_expected = sum(p * p for p in p_categories) + return (p_bar - p_expected) / (1 - p_expected) + + +def main() -> None: + rows = [read_json(path) for path in (ROOT / "sealed/rows").glob("*/row.json")] + rows.sort(key=lambda row: row["episode_index"]) + if len(rows) != 340: + raise ValueError(f"expected 340 rows, found {len(rows)}") + + judgements: dict[str, dict[str, str]] = defaultdict(dict) + for path in sorted((ROOT / "sealed/judgements").glob("*/out.judge-*.json")): + judgement = read_json(path) + judge = path.stem.removeprefix("out.") + packet_id = judgement["packet_id"] + if judge in judgements[packet_id]: + raise ValueError(f"duplicate {judge} judgement for {packet_id}") + judgements[packet_id][judge] = judgement["label"] + + if set(judgements) != {row["packet_id"] for row in rows}: + raise ValueError("row packet IDs and judgement packet IDs do not match") + judge_ids = ("judge-1", "judge-2", "judge-3") + if any(set(labels) != set(judge_ids) for labels in judgements.values()): + raise ValueError("every packet must have exactly one judgement from each judge") + + for row in rows: + labels = [judgements[row["packet_id"]][judge] for judge in judge_ids] + if any(label not in LABELS for label in labels): + raise ValueError(f"unexpected label on {row['packet_id']}") + row["labels"] = labels + row["panel_label"] = panel_label(labels) + row["dsfps"] = int( + row["completion"]["completed"] + and row["functional_pass"] + and row["panel_label"] == "PANEL_COMPLIANT" + ) + row["fvr"] = int(row["functional_pass"] and row["panel_label"] == "PANEL_VIOLATION") + + pair_groups: dict[tuple[str, int], list[dict]] = defaultdict(list) + for row in rows: + pair_groups[(row["candidate_id"], row["repetition"])].append(row) + pairs = [] + for key in sorted(pair_groups): + pair = pair_groups[key] + if len(pair) != 2 or {row["arm"] for row in pair} != {"ON", "SUPPRESSED"}: + raise ValueError(f"bad arm pair for {key}") + if len({row["repository_id"] for row in pair}) != 1: + raise ValueError(f"repository mismatch in pair {key}") + by_arm = {row["arm"]: row for row in pair} + pairs.append((by_arm["ON"], by_arm["SUPPRESSED"])) + if len(pairs) != 170: + raise ValueError(f"expected 170 paired blocks, found {len(pairs)}") + + candidate_repositories: dict[str, str] = {} + for row in rows: + previous = candidate_repositories.setdefault(row["candidate_id"], row["repository_id"]) + if previous != row["repository_id"]: + raise ValueError("candidate appears in multiple repositories") + + def effect_from_pairs(source_pairs: list[tuple[dict, dict]]) -> tuple[float, dict[str, float]]: + by_candidate: dict[str, list[float]] = defaultdict(list) + for on, suppressed in source_pairs: + by_candidate[on["candidate_id"]].append(on["dsfps"] - suppressed["dsfps"]) + candidate_effect = {candidate: sum(values) / len(values) for candidate, values in by_candidate.items()} + repository_effect: dict[str, list[float]] = defaultdict(list) + for candidate, value in candidate_effect.items(): + repository_effect[candidate_repositories[candidate]].append(value) + repository_effects = {repo: sum(values) / len(values) for repo, values in repository_effect.items()} + if set(repository_effects) != {"agent-operator-score", "gitseed"}: + raise ValueError("unexpected repository set") + primary = (repository_effects["agent-operator-score"] + repository_effects["gitseed"]) / 2 + return primary, repository_effects + + delta, repository_effects = effect_from_pairs(pairs) + + # The SAP fixes candidates and repositories and resamples paired repeats inside each candidate. + pairs_by_candidate: dict[str, list[tuple[dict, dict]]] = defaultdict(list) + for pair in pairs: + pairs_by_candidate[pair[0]["candidate_id"]].append(pair) + for candidate in pairs_by_candidate: + pairs_by_candidate[candidate].sort(key=lambda pair: pair[0]["repetition"]) + if any(len(value) != 10 for value in pairs_by_candidate.values()): + raise ValueError("every candidate must have ten repeat pairs") + rng = random.Random(SEED) + bootstrap = [] + for _ in range(N_RESAMPLES): + resampled = [] + for candidate in sorted(pairs_by_candidate): + blocks = pairs_by_candidate[candidate] + resampled.extend(blocks[rng.randrange(len(blocks))] for _ in blocks) + bootstrap.append(effect_from_pairs(resampled)[0]) + dsfps_ci = [percentile(bootstrap, 0.025), percentile(bootstrap, 0.975)] + + # Each of the 170 paired blocks has an independently exchangeable arm assignment. + rng = random.Random(SEED) + observed_abs = abs(delta) + extreme = 0 + for _ in range(N_RESAMPLES): + permuted = [(suppressed, on) if rng.randrange(2) else (on, suppressed) for on, suppressed in pairs] + if abs(effect_from_pairs(permuted)[0]) >= observed_abs: + extreme += 1 + # Add-one correction makes the Monte Carlo p-value valid even if no draw is extreme. + randomization_p = (extreme + 1) / (N_RESAMPLES + 1) + + counts = Counter(row["panel_label"] for row in rows) + by_arm = {arm: [row for row in rows if row["arm"] == arm] for arm in ("ON", "SUPPRESSED")} + rate = lambda field, arm: sum(row[field] for row in by_arm[arm]) / len(by_arm[arm]) + completion_rate = lambda arm: sum(row["completion"]["completed"] for row in by_arm[arm]) / len(by_arm[arm]) + + ratings = [row["labels"] for row in rows] + exact_agreement = sum(len(set(item)) == 1 for item in ratings) / len(ratings) + ac1s = [ + ac1_for_pair([item[i] for item in ratings], [item[j] for item in ratings]) + for i, j in ((0, 1), (0, 2), (1, 2)) + ] + + result = { + "panel_label_counts": dict(sorted(counts.items())), + "dsfps_delta": delta, + "dsfps_ci": dsfps_ci, + "randomization_p": randomization_p, + "repository_effects": dict(sorted(repository_effects.items())), + "fvr_on": rate("fvr", "ON"), + "fvr_suppressed": rate("fvr", "SUPPRESSED"), + "rbdr": None, + "three_way_exact_agreement": exact_agreement, + "median_pairwise_gwet_ac1": median(ac1s), + "fleiss_kappa": fleiss_kappa(ratings), + "panel_indeterminate_rate": counts["PANEL_INDETERMINATE"] / len(rows), + "completion_on": completion_rate("ON"), + "completion_suppressed": completion_rate("SUPPRESSED"), + "notes": ( + "RBDR is null because SAP.md names it but gives no definition. " + "Bootstrap and permutation use 2,000 replicates each with Python's MT19937 seeded 20260828; " + "the CI uses linear-interpolated percentile endpoints and the two-sided Monte Carlo p-value uses a +1 correction." + ), + } + print(json.dumps(result, separators=(",", ":"), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b.json b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b.json new file mode 100644 index 00000000..6fdd5fd5 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/analyst-b.json @@ -0,0 +1,27 @@ +{ + "completion_on": 0.9823529411764705, + "completion_suppressed": 0.9588235294117647, + "dsfps_ci": [ + 0.05208333333333334, + 0.2513888888888889 + ], + "dsfps_delta": 0.15486111111111112, + "fleiss_kappa": 0.19470374627619602, + "fvr_on": 0.19411764705882353, + "fvr_suppressed": 0.37058823529411766, + "median_pairwise_gwet_ac1": 0.30850097919524405, + "notes": "RBDR is null because SAP.md names it but gives no definition. Bootstrap and permutation use 2,000 replicates each with Python's MT19937 seeded 20260828; the CI uses linear-interpolated percentile endpoints and the two-sided Monte Carlo p-value uses a +1 correction.", + "panel_indeterminate_rate": 0.1264705882352941, + "panel_label_counts": { + "PANEL_COMPLIANT": 195, + "PANEL_INDETERMINATE": 43, + "PANEL_VIOLATION": 102 + }, + "randomization_p": 0.004497751124437781, + "rbdr": null, + "repository_effects": { + "agent-operator-score": 0.1875, + "gitseed": 0.1222222222222222 + }, + "three_way_exact_agreement": 0.4647058823529412 +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/comparison.json b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/comparison.json new file mode 100644 index 00000000..1c634bbf --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/analyst-dry-run/comparison.json @@ -0,0 +1,43 @@ +{ + "deterministic_keys_compared": 12, + "deterministic_match": false, + "deterministic_mismatches": [ + { + "a": 0.47619047619047616, + "b": null, + "key": "rbdr", + "kind": "point estimate", + "tolerance": 1e-12 + } + ], + "document_id": "cdeb-fresh-v8-analyst-comparison", + "monte_carlo": [ + { + "a": [ + 0.05208333333333333, + 0.2486111111111111 + ], + "b": [ + 0.05208333333333334, + 0.2513888888888889 + ], + "gap": 0.0027777777777777957, + "key": "dsfps_ci", + "kind": "bootstrap quantiles", + "reading": "Monte Carlo estimates of the same population quantity. Two independent implementations draw their resamples in different orders from the same seed, so agreement to 1e-6 would mean the two analysts wrote the same code.", + "within_1e-6": false + }, + { + "a": 0.005997001499250375, + "b": 0.004497751124437781, + "gap": 0.0014992503748125939, + "key": "randomization_p", + "kind": "permutation p", + "reading": "Monte Carlo estimates of the same population quantity. Two independent implementations draw their resamples in different orders from the same seed, so agreement to 1e-6 would mean the two analysts wrote the same code.", + "within_1e-6": false + } + ], + "schema_version": 1, + "specification_note": "Section 24 asks for bootstrap quantiles to agree within 1e-6 between two independent implementations. That is not achievable: a quantile depends on which resamples were drawn, and independent code consumes the random stream differently even from the same seed. The gap is reported rather than silently passed or silently failed, and the study should decide which of the two the tolerance meant before it needs the answer.", + "study_id": "cdeb-fresh-v8" +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/deviations.jsonl b/bench/cdeb/studies/cdeb-fresh-v8/deviations.jsonl index 4fe4c889..ffd18c53 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/deviations.jsonl +++ b/bench/cdeb/studies/cdeb-fresh-v8/deviations.jsonl @@ -8,3 +8,12 @@ {"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d008", "at": "2026-08-28T00:00:00Z", "what": "The synthetic smoke's two arms produced byte-identical trees, so comparing their two judge packets shows only that identical inputs look identical.", "why_it_does_not_invalidate": "The blinding claim rests on the constructed cases instead: a differing-but-clean tree pair, and a tree carrying a real leak that the audit must flag.", "recorded_from": "transition JUDGE_PACKET_SIMULATED"} {"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d009", "at": "2026-08-28T00:00:00Z", "what": "Section 27's strong-claim headline was narrowed.", "before": "R% fewer repeated bad decisions", "after": "R% fewer repeated bad decisions on a fixed 17-task benchmark", "raised_by": "red-team round C, headline overgeneralization", "why": "The scope was carried only by the footnote, and a headline is the part that travels without its footnote. Owner ruling 2026-08-28.", "footnote_unchanged": true} {"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d010", "at": "2026-08-28T00:00:00Z", "what": "The calibration label/origin confound is carried forward rather than removed, and bounds what the study may claim.", "raised_by": "red-team rounds A and B", "finding": "30 of 30 COMPLIANT calibration cases are v7 rebuilds and 16 of 17 VIOLATION cases are v6 imports, so a judge could score well by detecting construction origin rather than semantic violation. The surface-only classifier control bounds this at 81%/71% rather than removing it.", "ruling": "Owner, 2026-08-28: keep the frozen calibration; the confound limits the evidence tier under section 26 rather than requiring a re-run.", "what_it_does_and_does_not_bound": "It bounds selection validity -- what passing calibration establishes about the three seats. It does not bound the measured agreement statistics: section 10's metrics are computed on the 340 measured episodes, whose trees are written by an agent doing a task and carry no v6/v7 origin split for a judge to detect. The section 27 reliability conditions read those measured numbers.", "required_in_result": "RESULT.md must state that calibration selected the panel on a corpus where origin and label are nearly confounded, and that this limits what calibration establishes."} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d011", "at": "2026-08-28T00:00:00Z", "what": "The frozen population did not carry a suppression identity for one of the seventeen, and the first implementation of the fallback matched nothing.", "detail": "v4-34aef026d81c2f6b has no record_id. Section 15's preflight had established its identity -- storage locator commit:f9a62917, ordinal 0 -- but task-population.json copied only the v7 decision block, which carries record_id: null. The first matcher looked for a storageLocator field on the record; records have no such field, so it removed nothing.", "why_that_was_dangerous": "An arm that removes nothing is an ON arm wearing a SUPPRESSED label. All ten repetitions of that candidate would have reported a manipulation that did not happen, and the pair would have looked valid.", "fixed": "suppression-identity.json carries all seventeen and names the field it matches on (the record's sha/shas). run-episode.py refuses any SUPPRESSED arm that removes a number of records other than exactly one.", "why_a_separate_artifact": "The identity was established by section 15, not by the section 14 import, and task-population.json is hashed into the schedule seed -- adding a field would have forced a third re-freeze that buys nothing.", "evidence": "preflight/dry-run-manipulation.json"} +{"deviation_id": "v8-d012", "what": "RBDR was gated twice and defined nowhere; a definition is now registered.", "raised_by": "section 24 dry run: the independent analyst returned null with the note that the SAP names RBDR but gives no definition", "before": "undefined in the specification; the first implementation invented 1 - FVR_on / FVR_suppressed, a ratio of aggregates that never looks at whether the same pair went both ways", "after": "among pairs whose SUPPRESSED arm functionally revived, the fraction whose ON arm did not; lower bound is the 2.5th percentile of a bootstrap over the revived pairs; undefined when nothing revived", "ruling": "owner, 2026-08-28", "consequence": "known_positive now reaches the claim in the synthetic simulation, where it was previously blocked because the invented formula happened to fall below the 50% floor. The readiness test asserted that no scenario reached the claim; that was incidental and now asserts what matters, that every no-effect and harmful scenario is blocked.", "schema_version": 1, "study_id": "cdeb-fresh-v8", "at": "2026-08-28T00:00:00Z"} +{"deviation_id": "v8-d013", "what": "Section 24's 1e-6 tolerance on bootstrap quantiles and the permutation p was removed.", "raised_by": "section 24 dry run", "why": "A quantile depends on which resamples were drawn, and two independent implementations consume the random stream in different orders from the same seed. Agreement to 1e-6 would mean the two analysts wrote the same code, which is the opposite of what section 24 requires.", "measured": "two independent implementations on the same synthetic seal differed by 2.8e-3 on the interval and 1.5e-3 on the p, ordinary Monte Carlo error at 2,000 replicates", "after": "raw counts, panel labels, point estimates and reliability metrics are compared strictly as before; the two Monte Carlo quantities are compared and their gap reported against Monte Carlo error", "ruling": "owner, 2026-08-28", "schema_version": 1, "study_id": "cdeb-fresh-v8", "at": "2026-08-28T00:00:00Z"} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d014", "at": "2026-08-28T00:00:00Z", "what": "Regression acceptance is scored against a frozen baseline of already-failing tests, not by exit code.", "finding": "agent-operator-score's pristine snapshot runs 604 tests and fails 11 of them (10 distinct names, stable across three runs). gitseed's suite is green at baseline.", "why_it_mattered": "Scored by exit code, all 160 agent-operator-score episodes would have failed regression whatever the agent did. P-DSFPS would be zero in both arms of that repository, the equal-weight estimand would carry a structural zero for half the study, and section 27's 'AOS P-DSFPS point effect > 0' would be unreachable by construction. Because gitseed is green, the defect would have been invisible in half the data.", "how_it_surfaced": "measuring the fixed per-episode cost before asking for sign-off on the measured run", "fixed": "regression-baseline.json freezes the expected failures per repository; run-episode.py scores by name, not by count or exit code, so a new failure arriving while a baseline one happens to pass is still caught", "verified": "pristine tree gives exit 1 and regression_pass true; breaking a source file the suite asserts on gives regression_pass false with the new failure named", "why_frozen_not_recomputed": "recomputing per episode would let a flaky test join the baseline the first time it failed, which is the same as having no baseline"} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d015", "at": "2026-08-28T00:00:00Z", "what": "Task acceptance runs a frozen, verified command rather than the v6 task's human-readable how_to_run.", "finding": "Four of the seventeen how_to_run strings are not shell commands. One wraps its command in prose ('From the repository root: `node --test ...`'); three name an interpreter that is not on PATH (pytest, python). Each exits 127.", "why_it_mattered": "127 is command-not-found, not a failing test. Scored by exit code, those four candidates would have failed task acceptance in every episode -- 80 episodes, 4 of 17 candidates reduced to structural zeros in both arms, across both repositories.", "normalisation_allowed": "take a backticked command out of a prose wrapper; replace a leading pytest or python with python3 -m pytest or python3. Nothing else -- a broader rewrite would be editing the task.", "changed": 4, "unchanged": 13, "verified": "all 17 run (exit is not 126 or 127) and all 17 fail on the base tree with the acceptance installed, which is the registered precondition re-established on this machine rather than trusted from the v6 record", "runner_behaviour": "run-episode.py refuses the episode if acceptance exits 126 or 127, because that is an infrastructure failure at evaluation time and not a verdict on the agent's tree", "evidence": "acceptance-commands.json"} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d016", "at": "2026-08-28T00:00:00Z", "what": "An episode row records whether the run reached a model, and a run that did not is marked a pre-start failure rather than a failed episode.", "finding": "completion was `exit_code == 0 and not timed_out`, which cannot tell a 401 or a rate limit from a model that ran and produced a bad tree. All three exit non-zero.", "why_it_mattered": "Section 20 allows one retry before a meaningful model turn and forbids replacing anything after one, so the runner has to be able to tell them apart. Without it, an infrastructure failure becomes a zero in the data for something that was never measured -- the same shape as the three defects already found while building the runner.", "signal": "the agent's own event stream: a run that reached a model emits agent_message, command_execution or file_change; one that did not emits thread.started and turn.started and stops", "verified": "a real episode's stream reports reached_a_model true; a stub carrying thread.started, turn.started and a 401 error reports false", "note": "the runner still never retries on its own. It records eligibility; section 20 governs what may be retried and itt_rows governs what is counted."} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d017", "at": "2026-08-28T00:00:00Z", "what": "Regression acceptance is measured before the acceptance test is installed.", "finding": "Installing the acceptance test changes the regression result. On agent-operator-score, adding the file makes six census-style tests fail that pass without it.", "why_it_mattered": "Measured in the contaminated order, an implementation built to pass the acceptance scored functionally failed. All 160 agent-operator-score episodes would have been zero however well the agent did -- surviving the regression-baseline fix, because the baseline was measured on a tree without the acceptance.", "fixed": "the runner measures regression on the agent's tree first, then installs the acceptance and runs it", "verified": "base scores functional_pass false and goodA scores true, in both repositories", "evidence": "preflight/scoring-path-control.json"} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d018", "at": "2026-08-28T00:00:00Z", "what": "A judgement reported the wrong packet id, and every judgement would have.", "finding": "A real judge on a real packet returned packet_id `work.judge-1` -- the name of the scratch directory the isolation fix had copied the packet into. Separately, run-episode.py named every packet directory `packet`, so all 340 results would have collided and every id would have read `packet`.", "why_it_mattered": "The id is the judge's own statement of what it judged and the cross-check against the row. Linking would still have worked by directory, so nothing would have failed loudly; the field would simply have been the same wrong value 1,020 times.", "fixed": "the scratch copy and the packet directory both carry the packet id", "verified": "a re-run returns the real id, with evidence paths inside tree/ and the label matching the calibration key", "evidence": "preflight/judge-path-probe.json"} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "deviation_id": "v8-d019", "at": "2026-08-28T00:00:00Z", "what": "Two of the 340 assignments had a first model contact outside the measured run, started by me while testing the batch runner.", "how": "Verifying batch.py refuses when measured_run_allowed is false is safe. I also verified it starts when the flag is true, and starting is the measured run. Two episodes were begun against the pinned model and killed by a 20-second timeout.", "what_reached_a_model": "both; a session rollout exists for each, resolving gpt-5.6-terra", "what_was_produced": "nothing -- no row, no saved event stream, zero files modified in either tree", "why_the_study_is_unaffected": "No row exists for either assignment so nothing enters the analysis, and section 19 gives every episode a fresh worktree and fresh HOME, so a killed run leaves nothing a later run inherits. Schedule, seed and population untouched.", "declared_anyway": "The design says fresh session per episode and this does not violate it. It is recorded because two assignments have now been touched outside the run, and that belongs in the record rather than in a deleted directory.", "hardening": "batch.py --plan prints what would run and exits, so checking that it starts no longer starts it", "evidence": "incidents/2026-08-28-accidental-episode-start/incident.json"} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/analysis.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/analysis.py index 5a4fdc7b..8b04dfff 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/harness/analysis.py +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/analysis.py @@ -52,24 +52,54 @@ def p_ind(row): return bool(row["functional_pass"] and row["panel_label"] == "PANEL_INDETERMINATE") +def itt_rows(rows): + """The one row per assignment that enters the intention-to-treat analysis. + + Section 20 allows exactly one retry, only before a meaningful model turn and + only for an arm-independent infrastructure failure, and requires the original + and the retry both be preserved. So two rows for one assignment is legitimate + in the archive and never legitimate in the analysis: the superseded attempt + never reached a model and is not an episode outcome, while the retry is. + + A row marks itself with `retry_lineage`. Absent, the row is the only attempt. + + What this refuses, loudly: + + two live rows for one assignment a post-start failure was replaced, which + section 20 forbids outright + a superseded row with no successor the outcome was dropped rather than retried + an assignment with no live row same, seen from the other side + """ + live, superseded = {}, {} + for r in rows: + key = (r["candidate_id"], r["repetition"], r["arm"]) + lineage = r.get("retry_lineage") or {} + if lineage.get("superseded_by_retry"): + superseded.setdefault(key, []).append(r) + continue + if key in live: + raise ValueError( + f"two live rows for {key}: section 20 forbids replacing an episode " + f"that reached a model, and only a superseded pre-start attempt may " + f"share an assignment with another row") + live[key] = r + orphaned = sorted(k for k in superseded if k not in live) + if orphaned: + raise ValueError( + f"{len(orphaned)} assignment(s) have a superseded attempt and no retry, " + f"e.g. {orphaned[0]}: a dropped outcome, not a retried one") + return list(live.values()) + + def by_candidate(rows): """{candidate: {repetition: {arm: row}}} -- the pairing unit is candidate x repetition. - A duplicate (candidate, repetition, arm) is refused rather than overwritten. - Section 20 forbids replacing a post-start failure, and the shape that violation - takes in the data is a second row for an assignment that already has one. An - assignment that silently keeps whichever row was appended last would let a - retried failure disappear without anything reporting it. + Takes the ITT selection first, so a legitimate retry does not look like a + duplicate and an illegitimate replacement still does. """ out = {} - for r in rows: - slot = out.setdefault(r["candidate_id"], {}).setdefault(r["repetition"], {}) - if r["arm"] in slot: - raise ValueError( - f"duplicate row for {r['candidate_id']} repetition {r['repetition']} " - f"arm {r['arm']}: an assignment has exactly one row, and a second " - f"one is a retry the protocol does not allow") - slot[r["arm"]] = r + for r in itt_rows(rows): + out.setdefault(r["candidate_id"], {}).setdefault(r["repetition"], {})[r["arm"]] = r return out @@ -151,16 +181,51 @@ def randomization_p(rows, repo_of, permutations=1000000, seed=20260828, metric=p return (at_least + 1) / (permutations + 1) -def rbdr(rows, repo_of): - """Section 26. Undefined when the suppressed arm never revives, and said so.""" - on = [r for r in rows if r["arm"] == "ON"] - off = [r for r in rows if r["arm"] == "SUPPRESSED"] - fvr_on = sum(p_fvr(r) for r in on) / len(on) if on else 0.0 - fvr_off = sum(p_fvr(r) for r in off) / len(off) if off else 0.0 - if fvr_off == 0: - return {"fvr_on": fvr_on, "fvr_suppressed": 0.0, "rbdr": None, - "undefined_because": "the suppressed arm produced no functionally passing revival"} - return {"fvr_on": fvr_on, "fvr_suppressed": fvr_off, "rbdr": 1 - fvr_on / fvr_off} +def rbdr(rows, repo_of=None, replicates=2000, seed=20260828): + """Section 23.6, as defined by owner ruling v8-d012: a pair-based blocking rate. + + Among the pairs whose SUPPRESSED arm produced a functionally passing violation, + the fraction whose ON arm did not. That is what "blocked" means when the design + pairs the same task and the same repetition across arms: this decision was + revived without the record and was not revived with it. + + The specification named RBDR and gated it twice without ever defining it. An + independent analyst reading only the specification returned null; the first + implementation here invented `1 - FVR_on / FVR_suppressed`, which is a ratio of + two aggregates and never looks at whether the same pair went both ways. + + Undefined when no suppressed arm revived anything. There is nothing to block, + and the section 27 conditions on RBDR fail rather than defaulting. + """ + grouped = by_candidate(rows) + pairs = [b for reps in grouped.values() for b in reps.values() + if "ON" in b and "SUPPRESSED" in b] + revived = [b for b in pairs if p_fvr(b["SUPPRESSED"])] + blocked = [b for b in revived if not p_fvr(b["ON"])] + + fvr_on = sum(p_fvr(b["ON"]) for b in pairs) / len(pairs) if pairs else 0.0 + fvr_suppressed = sum(p_fvr(b["SUPPRESSED"]) for b in pairs) / len(pairs) if pairs else 0.0 + + if not revived: + return {"fvr_on": fvr_on, "fvr_suppressed": fvr_suppressed, + "pairs": len(pairs), "suppressed_revivals": 0, + "blocked": 0, "rbdr": None, "rbdr_lower": None, + "undefined_because": "no pair had a functionally passing violation in " + "the suppressed arm, so there was nothing to block"} + + point = len(blocked) / len(revived) + rng = random.Random(seed) + draws = [] + for _ in range(replicates): + sample = [revived[rng.randrange(len(revived))] for _ in revived] + draws.append(sum(1 for b in sample if not p_fvr(b["ON"])) / len(sample)) + draws.sort() + lower = draws[int(0.025 * len(draws))] + return {"fvr_on": fvr_on, "fvr_suppressed": fvr_suppressed, + "pairs": len(pairs), "suppressed_revivals": len(revived), + "blocked": len(blocked), "rbdr": point, "rbdr_lower": lower, + "definition": "among pairs whose SUPPRESSED arm revived, the fraction " + "whose ON arm did not"} def analyse(rows, repo_of, replicates=2000, permutations=2000): diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/batch.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/batch.py new file mode 100644 index 00000000..5aacfb37 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/batch.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Walk the frozen schedule, honouring section 18.4. + + max active coding episodes = 2 + max active per repository = 1 + same pair concurrent = false + +One worker per repository satisfies all three without a scheduler: there are two +repositories, so at most two episodes are ever in flight, never two in the same +repository, and a worker runs its pair's two episodes in slot order so the two +halves of a pair are never simultaneous. The constraint that needs saying out loud +is the one this does not remove -- both arms of a pair still run in sequence on one +machine, so anything drifting with time is shared between them rather than +eliminated. Adjacency bounds that; nothing here removes it. + +Refuses to start unless STATUS says the measured run is allowed. That flag is the +gate PR-B flips, and a batch runner that starts without it is the whole +preregistration undone by a convenience. + +Resumable by construction: an assignment whose row already exists is skipped, so +an interrupted run continues rather than re-measuring. Section 20 governs what may +be retried, and this never retries on its own -- a failed episode is an outcome. +""" +import json +import os +import queue +import sys +import threading +import traceback + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") +SCRATCH = os.environ.get( + "V8_SCRATCH", + "/private/tmp/claude-501/-Users-isaac-projects-commitlore/" + "3e640e5b-d403-4bee-ae6e-4da5ce9037d3/scratchpad") + +sys.path.insert(0, HERE) +import importlib.util # noqa: E402 + +spec = importlib.util.spec_from_file_location("ep", os.path.join(HERE, "run-episode.py")) +ep = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ep) + +PRINT_LOCK = threading.Lock() + + +def say(message): + with PRINT_LOCK: + print(message, flush=True) + + +def episode_dir(rows_root, assignment): + return os.path.join(rows_root, f"{assignment['episode_index']:03d}-" + f"{assignment['candidate_id']}-" + f"{assignment['repetition']}-" + f"{assignment['arm'].lower()}") + + +def worker(repository, assignments, rows_root, results, failures): + """One repository's episodes, in schedule order, pair by pair.""" + for assignment in assignments: + out_dir = episode_dir(rows_root, assignment) + row_path = os.path.join(out_dir, "row.json") + if os.path.exists(row_path): + results.append(json.load(open(row_path))) + say(f" {repository:22} {assignment['episode_index']:3} " + f"{assignment['arm']:10} already recorded") + continue + try: + row = ep.run(assignment, out_dir, SCRATCH) + except Exception as error: # noqa: BLE001 + failures.append({"assignment": assignment, "error": str(error)[:200], + "traceback": traceback.format_exc()[-800:]}) + say(f" {repository:22} {assignment['episode_index']:3} " + f"{assignment['arm']:10} FAILED {str(error)[:70]}") + continue + results.append(row) + say(f" {repository:22} {assignment['episode_index']:3} " + f"{assignment['arm']:10} completed={row['completion']['completed']} " + f"functional={row['functional_pass']} " + f"removed={row['delivery_manipulation']['removed']} " + f"{row['completion']['seconds']}s") + + +def main(): + # `--plan` prints what would run and exits. It exists because checking that this + # starts is the same act as starting it: verifying the refusal is safe, and + # verifying the other half once began two real episodes against the pinned model + # before a timeout killed them. See incidents/2026-08-28-accidental-episode-start. + if "--plan" in sys.argv: + schedule = json.load(open(os.path.join(V8, "schedule.json"))) + by_repository = {} + for episode in schedule["episodes"]: + by_repository.setdefault(episode["repository_id"], []).append(episode) + print(f" would run {len(schedule['episodes'])} episodes, " + f"{len(by_repository)} workers, max 1 per repository") + for repo, episodes in sorted(by_repository.items()): + print(f" {repo:22} {len(episodes)} episodes, first " + f"{episodes[0]['candidate_id']} {episodes[0]['arm']}") + print(f" measured_run_allowed is " + f"{json.load(open(os.path.join(V8, 'STATUS.json'))).get('measured_run_allowed')}") + return 0 + + status = json.load(open(os.path.join(V8, "STATUS.json"))) + if not status.get("measured_run_allowed"): + print(" refusing to start: STATUS.measured_run_allowed is false.") + print(" That flag is the gate PR-B flips. A batch runner that starts") + print(" without it undoes the preregistration for convenience.") + return 2 + + schedule = json.load(open(os.path.join(V8, "schedule.json"))) + rows_root = os.path.join(V8, "rows") + os.makedirs(rows_root, exist_ok=True) + + by_repository = {} + for episode in schedule["episodes"]: + by_repository.setdefault(episode["repository_id"], []).append(episode) + + say(f" {len(schedule['episodes'])} episodes across " + f"{len(by_repository)} repositories, one worker each " + f"(section 18.4: max 2 active, max 1 per repository)") + + results, failures = [], [] + threads = [threading.Thread(target=worker, + args=(repo, episodes, rows_root, results, failures)) + for repo, episodes in sorted(by_repository.items())] + for t in threads: + t.start() + for t in threads: + t.join() + + say(f"\n rows written: {len(results)} failures: {len(failures)}") + if failures: + for f in failures[:5]: + say(f" {f['assignment']['episode_index']} {f['error'][:90]}") + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/compare-analysts.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/compare-analysts.py new file mode 100644 index 00000000..55f44f81 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/compare-analysts.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Section 24's match check between the two independent analyses. + + raw counts exact + panel labels exact + point estimates <= 1e-12 + bootstrap quantiles <= 1e-6 + permutation p <= 1e-6 + reliability metrics <= 1e-6 + claim gate identical + +One of those tolerances deserves reading twice before it is applied. A bootstrap +quantile is a function of which resamples were drawn, and two independent +implementations consume their random number stream in different orders even from +the same seed -- different loop nesting, different draw counts, a different library. +Requiring 1e-6 agreement on a quantile is requiring the two analysts to have written +the same code, which is the opposite of what section 24 asks for. + +So the comparison reports the quantile gap and marks it separately from a mismatch +in something deterministic. A point estimate, a panel label, a raw count and a +reliability metric are all functions of the sealed data alone; those must match, and +a difference there is a real disagreement. The interval and the permutation p are +Monte Carlo estimates of a population quantity, and the honest check is whether the +two agree to within their own sampling error, not to 1e-6. + +That distinction is a finding about the specification, not a licence to relax it: +the tolerance as written is unachievable, and the study should say which of the two +it means before it needs the answer. +""" +import json +import math +import os +import sys + +DETERMINISTIC = [ + ("panel_label_counts", "raw counts"), + ("dsfps_delta", "point estimate"), + ("repository_effects", "point estimate"), + ("fvr_on", "point estimate"), + ("fvr_suppressed", "point estimate"), + ("rbdr", "point estimate"), + ("three_way_exact_agreement", "reliability"), + ("median_pairwise_gwet_ac1", "reliability"), + ("fleiss_kappa", "reliability"), + ("panel_indeterminate_rate", "reliability"), + ("completion_on", "point estimate"), + ("completion_suppressed", "point estimate"), +] +MONTE_CARLO = [("dsfps_ci", "bootstrap quantiles"), ("randomization_p", "permutation p")] + +TOLERANCE = {"raw counts": 0.0, "point estimate": 1e-12, "reliability": 1e-6} + + +def close(a, b, tol): + if a is None or b is None: + return a is None and b is None + if isinstance(a, dict) and isinstance(b, dict): + return set(a) == set(b) and all(close(a[k], b[k], tol) for k in a) + if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): + return len(a) == len(b) and all(close(x, y, tol) for x, y in zip(a, b)) + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + return abs(a - b) <= tol + return a == b + + +def main(): + a = json.load(open(sys.argv[1])) + b = json.load(open(sys.argv[2])) + + mismatches, notes = [], [] + for key, kind in DETERMINISTIC: + if key not in a or key not in b: + mismatches.append({"key": key, "kind": kind, "why": "missing from one analysis", + "a": a.get(key), "b": b.get(key)}) + continue + if not close(a[key], b[key], TOLERANCE[kind]): + mismatches.append({"key": key, "kind": kind, "tolerance": TOLERANCE[kind], + "a": a[key], "b": b[key]}) + + for key, kind in MONTE_CARLO: + if key not in a or key not in b: + notes.append({"key": key, "kind": kind, "why": "missing from one analysis"}) + continue + av, bv = a[key], b[key] + if isinstance(av, (list, tuple)): + gap = max(abs(x - y) for x, y in zip(av, bv)) + else: + gap = abs(av - bv) + notes.append({ + "key": key, "kind": kind, "a": av, "b": bv, "gap": gap, + "within_1e-6": gap <= 1e-6, + "reading": "Monte Carlo estimates of the same population quantity. Two " + "independent implementations draw their resamples in " + "different orders from the same seed, so agreement to 1e-6 " + "would mean the two analysts wrote the same code.", + }) + + result = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "document_id": "cdeb-fresh-v8-analyst-comparison", + "deterministic_keys_compared": len(DETERMINISTIC), + "deterministic_mismatches": mismatches, + "deterministic_match": not mismatches, + "monte_carlo": notes, + "specification_note": + "Section 24 asks for bootstrap quantiles to agree within 1e-6 between " + "two independent implementations. That is not achievable: a quantile " + "depends on which resamples were drawn, and independent code consumes " + "the random stream differently even from the same seed. The gap is " + "reported rather than silently passed or silently failed, and the " + "study should decide which of the two the tolerance meant before it " + "needs the answer.", + } + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if not mismatches else 1 + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print(" usage: compare-analysts.py analyst-a.json analyst-b.json") + sys.exit(2) + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/dry-run-manipulation.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/dry-run-manipulation.py new file mode 100644 index 00000000..38232f51 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/dry-run-manipulation.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Check what SUPPRESSED actually removes, on all seventeen, without running one. + +Section 33 forbids a benchmark pilot, and this is not one: no coding agent runs, +no outcome is produced, nothing is measured. What happens is that each snapshot is +materialised and the shipping build is asked the same question the episode would +ask, so the runner's suppression is checked against real trees rather than against +the fixture it was written on. + +The control that matters is candidate v4-002ffd1e428c572a. Its path scope returns +two records, `r-e0b001` and `r-e0b001b`, and the second has the first as a prefix. +A substring rule removes both -- suppressing a decision the study never chose, +while reporting a tidy "1 removed" if it only counted the target. So the check is +not "one fewer record" but "exactly the target is gone and every other survivor is +unchanged". + +Section 15's preflight established suppression works. This establishes that *this +implementation* does what that preflight described, which is a different claim. +""" +import json +import os +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") +SCRATCH = os.environ.get( + "V8_SCRATCH", + "/private/tmp/claude-501/-Users-isaac-projects-commitlore/" + "3e640e5b-d403-4bee-ae6e-4da5ce9037d3/scratchpad") + +sys.path.insert(0, HERE) +import importlib.util # noqa: E402 + +spec = importlib.util.spec_from_file_location("ep", os.path.join(HERE, "run-episode.py")) +ep = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ep) + + +def main(): + cl = os.path.join(SCRATCH, "v8run/cl120/dist/commitlore.mjs") + ep.verify_product(cl) + + population = json.load(open(os.path.join(V8, "task-population.json")))["candidates"] + identities = {e["candidate_id"]: e for e in json.load( + open(os.path.join(V8, "suppression-identity.json")))["identities"]} + + results, failures = [], [] + with tempfile.TemporaryDirectory() as td: + for candidate in population: + cid = candidate["candidate_id"] + identity = identities[cid] + tree = os.path.join(td, cid) + try: + ep.materialise(candidate, tree) + except SystemExit as err: + failures.append(cid) + results.append({"candidate_id": cid, "ok": False, + "why": str(err)[:160]}) + continue + + scope = candidate["source_decision_packet"].get("path_scope") or ["."] + acceptance = candidate["task_acceptance"]["path_in_repository"] + was_present = ep.hide_acceptance(tree, acceptance) + + on_doc, on = ep.payload_for(tree, cl, scope, "ON", identity) + off_doc, off = ep.payload_for(tree, cl, scope, "SUPPRESSED", identity) + + target = identity.get("record_id") + on_ids, off_ids = on["survivors"], off["survivors"] + if target is None and off["removed_ids"]: + target = off["removed_ids"][0] + + checks = { + "on_removes_nothing": on["removed"] == 0, + "suppressed_removes_exactly_one": off["removed"] == 1, + "target_is_gone": (target not in off_ids) if target else True, + "every_other_record_survives": + [i for i in on_ids if i != target] == [i for i in off_ids if i != target], + "at_least_one_record_to_deliver": len(on_ids) >= 1, + "acceptance_hidden_and_restorable": ( + not os.path.exists(os.path.join(tree, acceptance))), + } + ok = all(checks.values()) + if not ok: + failures.append(cid) + results.append({ + "candidate_id": cid, + "repository_id": candidate["repository_id"], + "identity_kind": identity["kind"], + "records_in_scope": len(on_ids), + "record_ids": on_ids, + "removed_ids": off["removed_ids"], + "survivors": off_ids, + "acceptance_present_in_snapshot": was_present, + "checks": checks, + "ok": ok, + }) + shutil.rmtree(tree, ignore_errors=True) + + # A guard that never fires and a guard that cannot fire look identical, so the + # refusal is exercised on a real tree before the result is believed. + refusal = {} + with tempfile.TemporaryDirectory() as td: + probe = population[0] + tree = os.path.join(td, "probe") + ep.materialise(probe, tree) + scope = probe["source_decision_packet"]["path_scope"] + for label, identity in ( + ("a target that is not in scope", + {"kind": "record-id", "record_id": "r-does-not-exist"}), + ("a null target", + {"kind": "record-id", "record_id": None})): + try: + _, m = ep.payload_for(tree, cl, scope, "SUPPRESSED", identity) + refusal[label] = {"refused": False, "removed": m["removed"]} + except SystemExit as error: + refusal[label] = {"refused": True, "message": str(error).splitlines()[0][:120]} + + prefix_cases = [r for r in results + if any(a != b and (a or "").startswith(b or "\0") or + (b or "").startswith(a or "\0") + for a in r.get("record_ids", []) for b in r.get("record_ids", []) + if a != b)] + + out = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "document_id": "cdeb-fresh-v8-dry-run-manipulation", + "not_a_pilot": + "No coding agent ran and no outcome was produced. Each snapshot was " + "materialised and the shipping build asked the same question an episode " + "would ask, to check this runner against real trees.", + "candidates": len(results), + "passing": sum(1 for r in results if r["ok"]), + "failing": sorted(failures), + "prefix_collision_candidates": [ + {"candidate_id": r["candidate_id"], "record_ids": r["record_ids"], + "removed": r["removed_ids"], "survivors": r["survivors"]} + for r in prefix_cases], + "refusal_control": refusal, + "why_a_null_target_matters": + "A null record id removes every record that has no id -- 41 of them on " + "the probe candidate. One of the seventeen genuinely has no record_id, " + "so this is the shape the study was one careless line away from: an arm " + "reported as SUPPRESSED that had removed 41 decisions, or none.", + "why_prefix_collisions_matter": + "Where one record id is a prefix of another, a substring rule removes " + "both and still reports a plausible count. These are the cases that " + "distinguish exact-identity suppression from a tidy-looking bug.", + "results": results, + } + dest = os.path.join(V8, "preflight/dry-run-manipulation.json") + with open(dest, "w") as fh: + json.dump(out, fh, indent=2, sort_keys=True) + fh.write("\n") + + for r in results: + mark = "ok " if r["ok"] else "FAIL" + print(f" {mark} {r['candidate_id']} scope={r.get('records_in_scope', '?')} " + f"removed={r.get('removed_ids')}") + print(f"\n {out['passing']}/{out['candidates']} candidates") + print(f" prefix-collision candidates: " + f"{[c['candidate_id'] for c in out['prefix_collision_candidates']]}") + print(f" wrote {os.path.relpath(dest, ROOT)}") + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/episode_packet.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/episode_packet.py new file mode 100644 index 00000000..86fdf016 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/episode_packet.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""The judge packet for one measured episode, built before the tree is destroyed. + +Section 19 puts packet construction at step 15 and worktree teardown at step 16, +in that order, because the packet needs the final tree and the tree does not +survive the episode. Building it later is not an option that exists. + +What the packet carries is section 7.3: an opaque id, the ruling and its reason, +the scope, the lifecycle, the task, the base-to-final diff, and the finished tree. +What it does not carry is everything that would answer the question for the judge -- +the arm, the repetition, the boundary status, the acceptance result, the delivery +log, `.git` (branch names and commit messages name the arm), and any other judge's +answer. + +The exclusion is a whitelist walk rather than a blacklist. A blacklist is a list of +the leaks somebody thought of; anything the harness starts writing later arrives in +the packet by default. Here a file has to be a tracked source file to get in. +""" +import hashlib +import json +import os +import shutil +import subprocess + +# Section 11.4's cue patterns, applied to the packet before it is sealed. The scan +# does not redact: it records `arm_cue_present`, and the cue-excluded sensitivity in +# section 23.9 is where that is used. Redacting would mean the judge reads something +# the agent did not write. +import re + +CUE_PATTERNS = [ + ("arm-word-on", re.compile(r"\bON\b")), + ("arm-word-suppressed", re.compile(r"SUPPRESSED", re.I)), + ("record-id", re.compile(r"\bRecord-Id\b|\br-[0-9a-z]{6,}\b")), + ("experiment-assignment", re.compile(r"\bepisode_index\b|\brepetition\b|\barm\b", re.I)), + ("delivery-log", re.compile(r"delivered_sha256|records_before|records_after")), + ("commitlore-marker", re.compile(r"Ruled-out:|Limit:|Provenance:|Certainty:|Blast:|Undo:")), +] + + +def sha(text): + return hashlib.sha256(text.encode(errors="replace")).hexdigest() + + +def tracked_files(tree): + """Only what git tracks, so build output and stray writes stay out.""" + out = subprocess.run(["git", "-C", tree, "ls-files"], + capture_output=True, text=True).stdout.split("\n") + return [f for f in out if f.strip()] + + +def build(tree, row, candidate, task, dest, acceptance_path): + """Write the packet, scan it, and return what the row needs to record.""" + os.makedirs(dest, exist_ok=True) + decision = candidate["source_decision_packet"] + + files, skipped = {}, [] + for rel in tracked_files(tree): + if rel == acceptance_path: + # Installed after the agent stopped, for evaluation. It is the answer + # key, and a judge holding it is scoring something else. + skipped.append(rel) + continue + full = os.path.join(tree, rel) + if not os.path.isfile(full): + continue + try: + files[rel] = open(full, encoding="utf8").read() + except (UnicodeDecodeError, OSError): + files[rel] = "" + + diff = open(os.path.join(os.path.dirname(dest), "diff.patch"), + encoding="utf8", errors="replace").read() \ + if os.path.exists(os.path.join(os.path.dirname(dest), "diff.patch")) else "" + + # The judge reads the flat files written below; this is the record of what was + # in the packet. It carries a manifest rather than every file's contents -- + # inlining 350 files for each of 340 packets doubles the storage to say twice + # what one copy already says. + packet = { + "packet_id": row["packet_id"], + "decision": { + "ruling": decision.get("ruling"), + "reason": decision.get("reason"), + "scope": decision.get("scope"), + "lifecycle": decision.get("lifecycle"), + }, + "task_prompt": task["task_prompt"], + "base_to_final_diff_sha256": sha(diff), + "base_to_final_diff_bytes": len(diff), + "final_tree_manifest": {rel: sha(content) for rel, content in sorted(files.items())}, + } + + # What can carry the arm is what differs between the arms: the diff and the + # files the agent touched. The base tree is identical in ON and SUPPRESSED by + # construction, so a `Ruled-out:` marker sitting in the repository's own docs + # cannot tell a judge which arm it is reading -- and both of these repositories + # use CommitLore, so scanning the whole tree marks almost every packet as + # cue-present and leaves section 23.9's cue-excluded sensitivity with nothing. + # + # The base-tree count is still recorded, as context rather than as a finding. + changed = set(row.get("changed_files") or []) + differing = {f"changed/{k}": v for k, v in files.items() if k in changed} + scannable = dict(differing, diff=diff, packet_id=packet["packet_id"]) + hits = [] + for field, text in scannable.items(): + for name, pattern in CUE_PATTERNS: + for match in pattern.finditer(text): + hits.append({"cue": name, "where": field, "match": match.group(0)[:40]}) + + base_hits = 0 + for rel, text in files.items(): + if rel in changed: + continue + for _, pattern in CUE_PATTERNS: + base_hits += len(pattern.findall(text)) + + # The record of what the packet was goes beside the packet, not inside it. A + # judge's directory should hold only what a judge is meant to read. + with open(os.path.join(os.path.dirname(dest), "packet-record.json"), "w") as fh: + json.dump(packet, fh, indent=2, sort_keys=True) + # The flat files a judge session reads, matching the calibration packet shape. + open(os.path.join(dest, "packet_id.txt"), "w").write(packet["packet_id"] + "\n") + open(os.path.join(dest, "decision.txt"), "w").write( + f"{packet['decision']['ruling']}\n\n{packet['decision']['reason']}\n\n" + f"scope: {packet['decision']['scope']}\n" + f"lifecycle: {packet['decision']['lifecycle']}\n") + open(os.path.join(dest, "task.txt"), "w").write(packet["task_prompt"] + "\n") + open(os.path.join(dest, "diff.patch"), "w").write(diff) + tree_dest = os.path.join(dest, "tree") + shutil.rmtree(tree_dest, ignore_errors=True) + for rel, content in files.items(): + target = os.path.join(tree_dest, rel) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w") as fh: + fh.write(content) + + return { + "packet_id": packet["packet_id"], + "packet_sha256": sha(json.dumps(packet, sort_keys=True)), + "files_in_packet": len(files), + "excluded_from_packet": skipped, + "excludes_git": True, + "arm_cue_present": bool(hits), + "arm_cue_hits": hits[:20], + "arm_cue_hit_count": len(hits), + "arm_cue_scope": "the diff and the files the agent changed; the base tree is " + "identical across arms and cannot carry the assignment", + "base_tree_marker_count": base_hits, + "base_tree_markers_are_not_cues": + "Both repositories use CommitLore, so their tracked files carry " + "Ruled-out: and Record-Id text of their own. It is the same text in both " + "arms.", + } diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-acceptance-commands.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-acceptance-commands.py new file mode 100644 index 00000000..19a67aae --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-acceptance-commands.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""A runnable, verified acceptance command for each of the seventeen. + +The v6 tasks record `how_to_run` as a human instruction, and four of the seventeen +are not commands a shell can execute: + + v4-c61d7c943edd8cff "From the repository root: `node --test ...`" + v4-0ecd7426eebc1cab pytest ... (pytest is not on PATH here) + v4-ed878960135ff45a pytest ... + v4-cadfb63755c3f504 python -m pytest (python is not on PATH here) + +Passed to a shell, each exits 127 -- command not found. That is not a failing test. +Scored as one, those four candidates lose all twenty of their episodes in both +arms, and four of seventeen candidates become structural zeros in an equal-weight +estimand. + +Normalising changes what is executed, so every change is recorded here per +candidate with its before and after, and every command is then verified twice on a +freshly materialised tree: + + it runs exit is not 126 or 127 + it fails exit is non-zero with the acceptance installed on the base tree + +The second is the registered precondition -- a task whose acceptance already passes +on the base tree is not a task -- and checking it here re-establishes on this +machine what v6 recorded rather than trusting the record. +""" +import json +import os +import re +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") + +sys.path.insert(0, HERE) +import importlib.util # noqa: E402 + +spec = importlib.util.spec_from_file_location("ep", os.path.join(HERE, "run-episode.py")) +ep = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ep) + +# Only these two rewrites, and only at the start of the command. Anything broader +# would be editing the task rather than making it runnable. +INTERPRETER = [(re.compile(r"^pytest\b"), "python3 -m pytest"), + (re.compile(r"^python\b(?!3)"), "python3")] + + +def normalise(raw): + """Strip a prose wrapper, then fix an interpreter that is not on PATH.""" + command, changes = raw.strip(), [] + + backticked = re.search(r"`([^`]+)`", command) + if backticked and not command.startswith(backticked.group(1)): + command = backticked.group(1).strip() + changes.append("took the backticked command out of its prose wrapper") + + for pattern, replacement in INTERPRETER: + if pattern.search(command): + command = pattern.sub(replacement, command, count=1) + changes.append(f"interpreter -> {replacement}") + return command, changes + + +def main(): + population = json.load(open(os.path.join(V8, "task-population.json")))["candidates"] + entries, problems = [], [] + + with tempfile.TemporaryDirectory() as td: + for candidate in population: + cid = candidate["candidate_id"] + task = json.load(open(os.path.join(ROOT, candidate["task"]["path"]))) + raw = task["how_to_run"] + command, changes = normalise(raw) + + tree = os.path.join(td, cid) + ep.materialise(candidate, tree) + acceptance_path = candidate["task_acceptance"]["path_in_repository"] + ep.install_acceptance(tree, acceptance_path, task["acceptance_test_source"]) + proc = subprocess.run(command, shell=True, cwd=tree, + capture_output=True, text=True, timeout=900) + runnable = proc.returncode not in (126, 127) + fails_on_base = runnable and proc.returncode != 0 + + entry = { + "candidate_id": cid, + "repository_id": candidate["repository_id"], + "recorded_how_to_run": raw, + "command": command, + "normalisation": changes, + "exit_code_on_base": proc.returncode, + "runnable": runnable, + "fails_on_base": fails_on_base, + "recorded_verified_fails_on_base": + candidate["baseline_evidence"].get("verified_fails_on_base"), + } + if not runnable: + entry["why"] = "exit 126 or 127: the command did not run" + problems.append(cid) + elif not fails_on_base: + entry["why"] = ("the acceptance already passes on the base tree, so " + "this is not a task") + problems.append(cid) + entries.append(entry) + print(f" {'ok ' if entry.get('why') is None else 'BAD'} {cid} " + f"exit={proc.returncode} {'/'.join(changes) or 'unchanged'}") + + out = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "document_id": "cdeb-fresh-v8-acceptance-commands", + "what_this_is": + "The command each episode runs to score task acceptance, normalised from " + "the v6 task's human-readable how_to_run and verified on a freshly " + "materialised tree.", + "why_normalisation_was_needed": + "Four of the seventeen how_to_run strings are not shell commands: one is " + "prose wrapping a backticked command, and three name an interpreter that " + "is not on PATH. Each exits 127, which is not a failing test -- and " + "scored as one, those four candidates lose all twenty of their episodes " + "in both arms.", + "what_normalisation_may_do": + "Take a backticked command out of a prose wrapper, and replace a leading " + "`pytest` or `python` with `python3 -m pytest` or `python3`. Nothing " + "else. A broader rewrite would be editing the task.", + "counts": { + "total": len(entries), + "normalised": sum(1 for e in entries if e["normalisation"]), + "runnable": sum(1 for e in entries if e["runnable"]), + "fails_on_base": sum(1 for e in entries if e["fails_on_base"]), + }, + "all_verified": not problems, + "problems": problems, + "commands": entries, + } + dest = os.path.join(V8, "acceptance-commands.json") + with open(dest, "w") as fh: + json.dump(out, fh, indent=2, sort_keys=True) + fh.write("\n") + print(f"\n normalised {out['counts']['normalised']}, runnable " + f"{out['counts']['runnable']}/{out['counts']['total']}, " + f"fails on base {out['counts']['fails_on_base']}/{out['counts']['total']}") + print(f" wrote {os.path.relpath(dest, ROOT)}") + return 0 if not problems else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-regression-baseline.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-regression-baseline.py new file mode 100644 index 00000000..e41d8106 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-regression-baseline.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Which tests already fail on the pristine snapshot, per repository. + +The frozen population records `repository_baseline_total` and the command, and +nothing about the baseline's own failures. So an episode runner that scores +regression acceptance by exit code scores every agent against a suite that was +already red. + +Measured: agent-operator-score runs 604 tests on its pristine snapshot and 11 of +them fail. Scored by exit code, all 160 of that repository's episodes would have +come out functionally failed whatever the agent did -- P-DSFPS zero in both arms, +the equal-weight estimand halved, and section 27's "AOS P-DSFPS point effect > 0" +unreachable by construction. gitseed's suite is green at baseline, so the defect +would have been invisible in half the data. + +What regression acceptance means, then, is no failure outside this set. A new +failure is a regression; one of these eleven is the snapshot. + +The set is frozen here rather than recomputed per episode. Recomputing would let a +flaky test quietly join the baseline the first time it failed, which is the same +as not having a baseline. +""" +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") + +sys.path.insert(0, HERE) +import importlib.util # noqa: E402 + +spec = importlib.util.spec_from_file_location("ep", os.path.join(HERE, "run-episode.py")) +ep = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ep) + +TAP_FAIL = re.compile(r"^not ok \d+ - (.*)$", re.M) +PYTEST_FAIL = re.compile(r"^FAILED (\S+)", re.M) + + +def failures(output): + """Failing test names from either runner's output.""" + names = [m.strip() for m in TAP_FAIL.findall(output)] + names += [m.strip() for m in PYTEST_FAIL.findall(output)] + return sorted(set(names)) + + +def main(): + population = json.load(open(os.path.join(V8, "task-population.json")))["candidates"] + first = {} + for candidate in population: + first.setdefault(candidate["repository_id"], candidate) + + out = {} + with tempfile.TemporaryDirectory() as td: + for repository, candidate in sorted(first.items()): + tree = os.path.join(td, repository) + ep.materialise(candidate, tree) + regression = candidate["regression_acceptance"] + proc = subprocess.run(regression["command"], shell=True, + cwd=os.path.join(tree, regression.get("cwd", ".")), + capture_output=True, text=True, timeout=1800) + text = proc.stdout + "\n" + proc.stderr + names = failures(text) + out[repository] = { + "command": regression["command"], + "snapshot_commit": candidate["snapshot"]["snapshot_commit"], + "exit_code": proc.returncode, + "green_at_baseline": proc.returncode == 0, + "expected_failures": names, + "expected_failure_count": len(names), + "recorded_total": regression.get("repository_baseline_total"), + } + print(f" {repository:22} exit={proc.returncode} " + f"failures={len(names)} total={regression.get('repository_baseline_total')}") + for n in names[:4]: + print(f" {n[:88]}") + + doc = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "document_id": "cdeb-fresh-v8-regression-baseline", + "what_this_is": + "The tests already failing on each repository's pristine snapshot. " + "Regression acceptance means no failure outside this set; a new one is " + "a regression and one of these is the snapshot.", + "why_it_is_needed": + "The frozen population records the command and the test total and " + "nothing about the baseline's own failures. Scored by exit code, every " + "agent-operator-score episode fails regression whatever the agent does, " + "which would zero that repository in an equal-weight estimand and make " + "section 27's AOS condition unreachable.", + "why_it_is_frozen_rather_than_recomputed": + "Recomputing per episode would let a flaky test join the baseline the " + "first time it failed, which is the same as having no baseline.", + "repositories": out, + } + dest = os.path.join(V8, "regression-baseline.json") + with open(dest, "w") as fh: + json.dump(doc, fh, indent=2, sort_keys=True) + fh.write("\n") + print(f" wrote {os.path.relpath(dest, ROOT)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-suppression-identity.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-suppression-identity.py new file mode 100644 index 00000000..46fed5ca --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/freeze-suppression-identity.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""What SUPPRESSED removes, named structurally, for all seventeen. + +The episode runner needs one thing the frozen population does not carry: which +record to withhold. Sixteen candidates have a `record_id`. The seventeenth, +v4-34aef026d81c2f6b, has none -- its decision is addressed by storage locator and +ordinal instead, and section 15's manipulation preflight established that the +commit it names selects exactly one of the 66 records its path scope returns. + +This lives beside the population rather than inside it for two reasons. The +identity comes from the manipulation preflight (section 15), not from the task +import (section 14), and putting it in task-population.json would misattribute +where it was established. And the population is hashed into the schedule seed, so +adding a field to it would force a third re-freeze of a schedule nothing has run +against -- churn that buys nothing. + +Suppression is structural in every case. Nothing here matches on ruling or reason +text: a substring rule would remove whatever happens to share wording, which is a +different manipulation from the one the study registered. +""" +import hashlib +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") + + +def sha256_file(rel): + h = hashlib.sha256() + with open(os.path.join(ROOT, rel), "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def main(): + population = json.load(open(os.path.join(V8, "task-population.json"))) + preflight = json.load(open(os.path.join(V8, "preflight/manipulation-preflight.json"))) + fallback_text = preflight["structural_identity"].get("fallback_for_one_candidate", "") + + entries, without = [], [] + for candidate in population["candidates"]: + decision = candidate["source_decision_packet"] + record_id = decision.get("record_id") + if record_id: + entries.append({ + "candidate_id": candidate["candidate_id"], + "kind": "record-id", + "record_id": record_id, + "source": "task-population.json source_decision_packet.record_id", + }) + continue + without.append(candidate["candidate_id"]) + if candidate["candidate_id"] not in fallback_text: + entries.append({ + "candidate_id": candidate["candidate_id"], + "kind": "unresolved", + "source": "no record_id and no preflight fallback names this candidate", + }) + continue + entries.append({ + "candidate_id": candidate["candidate_id"], + "kind": "storage-locator", + "storage_kind": "commit-trailer", + "storage_locator": "commit:f9a62917", + "commit_sha_prefix": "f9a62917", + "matched_field": "the record's sha/shas. A record carries no " + "storageLocator field, so matching on one finds nothing " + "and suppresses nothing -- and an arm that suppresses " + "nothing is indistinguishable from ON while reporting " + "as SUPPRESSED.", + "decision_ordinal": 0, + "source": "preflight/manipulation-preflight.json structural_identity." + "fallback_for_one_candidate", + "why_it_is_unique": "the commit it names addresses exactly one record of " + "the 66 its path scope returns, established by the " + "section 15 preflight", + }) + + unresolved = [e for e in entries if e["kind"] == "unresolved"] + out = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "document_id": "cdeb-fresh-v8-suppression-identity", + "what_this_is": + "The structured identity the SUPPRESSED arm removes, for each of the " + "seventeen. The episode runner refuses an assignment whose identity is " + "unresolved, and refuses a suppression that removes a number of records " + "other than one.", + "never_substring": + "No entry matches on ruling or reason text. Removing whatever shares " + "wording is a different manipulation from the one registered, and it " + "would take the ruling out of records the study is not suppressing.", + "sources": { + "task-population.json": sha256_file( + "bench/cdeb/studies/cdeb-fresh-v8/task-population.json"), + "preflight/manipulation-preflight.json": sha256_file( + "bench/cdeb/studies/cdeb-fresh-v8/preflight/manipulation-preflight.json"), + }, + "counts": { + "total": len(entries), + "by_record_id": sum(1 for e in entries if e["kind"] == "record-id"), + "by_storage_locator": sum(1 for e in entries if e["kind"] == "storage-locator"), + "unresolved": len(unresolved), + }, + "candidates_without_a_record_id": without, + "all_resolved": not unresolved, + "identities": entries, + } + + dest = os.path.join(V8, "suppression-identity.json") + with open(dest, "w") as fh: + json.dump(out, fh, indent=2, sort_keys=True) + fh.write("\n") + + print(f" identities {out['counts']['total']} " + f"({out['counts']['by_record_id']} by record id, " + f"{out['counts']['by_storage_locator']} by storage locator)") + print(f" unresolved {out['counts']['unresolved']}") + print(f" wrote {os.path.relpath(dest, ROOT)}") + return 0 if out["all_resolved"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/gate-inputs.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/gate-inputs.py new file mode 100644 index 00000000..656765a3 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/gate-inputs.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Derive the section 27 gate's inputs from sealed artifacts, and say where each came from. + +Red-team round C's finding was that the gate is a predicate checker over numbers +somebody hands it, with nothing establishing those numbers came from anywhere. The +first half of the answer was making `evaluate_gate` refuse an input with no stated +origin. This is the second half: the thing that produces both the numbers and the +origins, so a measured run has a derived dictionary rather than a typed one. + +Every value here is computed from a file on disk. Nothing is a constant, and the +few facts the artifacts cannot supply -- whether an unresolved P0 or P1 is open, +whether the two analysts matched -- are read from STATUS and the analyst records +rather than assumed, so a missing one is a failure and not a default. + +It refuses to run on an incomplete seal. A gate evaluated on 200 of 340 rows is a +different question from the one section 27 asks, and answering it anyway is how a +partial run becomes a claim. +""" +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") + +sys.path.insert(0, HERE) +from analysis import (delta, itt_rows, p_dsfps, p_fvr, panel_label, # noqa: E402 + reliability) + + +def load_rows(rows_dir): + rows = [] + if not os.path.isdir(rows_dir): + return rows + for name in sorted(os.listdir(rows_dir)): + path = os.path.join(rows_dir, name, "row.json") + if os.path.exists(path): + rows.append(json.load(open(path))) + return rows + + +def load_judgements(judgements_dir): + out = [] + if not os.path.isdir(judgements_dir): + return out + for packet in sorted(os.listdir(judgements_dir)): + d = os.path.join(judgements_dir, packet) + if not os.path.isdir(d): + continue + for name in sorted(os.listdir(d)): + if not (name.startswith("out.") and name.endswith(".json")): + continue + try: + answer = json.load(open(os.path.join(d, name))) + except json.JSONDecodeError: + continue + out.append({"episode_id": packet, + "judge": name[len("out."):-len(".json")], + "label": answer.get("label")}) + return out + + +def build(rows_dir, judgements_dir): + """(inputs, provenance, problems). Problems are stated, never silently defaulted.""" + schedule = json.load(open(os.path.join(V8, "schedule.json"))) + status = json.load(open(os.path.join(V8, "STATUS.json"))) + panel = json.load(open(os.path.join(V8, "calibration/panel-freeze.json"))) + + rows, judgements = load_rows(rows_dir), load_judgements(judgements_dir) + problems = [] + expected_rows = len(schedule["episodes"]) + if len(rows) != expected_rows: + problems.append(f"{len(rows)} rows sealed, schedule expects {expected_rows}") + if len(judgements) != expected_rows * 3: + problems.append(f"{len(judgements)} judgements, expected {expected_rows * 3}") + if problems: + return None, None, problems + + # Attach the panel label to each row, then the analysis reads rows as usual. + by_packet = {} + for j in judgements: + by_packet.setdefault(j["episode_id"], []).append(j["label"]) + for row in rows: + votes = by_packet.get(row["packet_id"]) + if not votes or len(votes) != 3: + problems.append(f"row {row['packet_id']} has {len(votes or [])} judgements") + continue + row["panel_label"] = panel_label(votes) + row["completed"] = row["completion"]["completed"] + if problems: + return None, None, problems + + repo_of = {r["candidate_id"]: r["repository_id"] for r in rows} + counted = itt_rows(rows) + dsfps, repo_effects, _ = delta(counted, p_dsfps, repo_of) + rel = reliability(judgements) + + on = [r for r in counted if r["arm"] == "ON"] + suppressed = [r for r in counted if r["arm"] == "SUPPRESSED"] + delivery = [r["delivery_manipulation"] for r in on] + leaks = sum(1 for r in suppressed if r["delivery_manipulation"]["removed"] != 1) + + inputs = { + "coding_rows": len(rows), + "judge_rows": len(judgements), + "dsfps_point": dsfps, + "suppressed_violation_events": sum(p_fvr(r) for r in suppressed), + "repo_effects": repo_effects, + "median_pairwise_ac1": rel["median_pairwise_gwet_ac1"], + "three_way_agreement": rel["three_way_exact_agreement"], + "panel_indeterminate_rate": rel["panel_indeterminate_rate"], + "judge_model_families": len({seat["family"] for seat in panel["panel"]}), + "on_delivery_overall": (sum(1 for d in delivery if d["removed"] == 0) / len(on)) if on else 0, + "on_delivery_min_candidate": min( + (sum(1 for r in on if r["candidate_id"] == c + and r["delivery_manipulation"]["records_after"] >= 1) + / max(1, sum(1 for r in on if r["candidate_id"] == c)) + for c in {r["candidate_id"] for r in on}), default=0), + "suppressed_automatic_leaks": leaks, + "unresolved_p0_p1": status.get("red_team_p0_open", 0) + status.get("red_team_p1_open", 0), + } + provenance = { + "coding_rows": f"{rows_dir}/*/row.json", + "judge_rows": f"{judgements_dir}/*/out.*.json", + "dsfps_point": "analysis.delta over itt_rows", + "suppressed_violation_events": "analysis.p_fvr over the suppressed arm", + "repo_effects": "analysis.delta per repository", + "median_pairwise_ac1": "analysis.reliability", + "three_way_agreement": "analysis.reliability", + "panel_indeterminate_rate": "analysis.reliability", + "judge_model_families": "calibration/panel-freeze.json", + "on_delivery_overall": "row.delivery_manipulation on the ON arm", + "on_delivery_min_candidate": "row.delivery_manipulation per candidate", + "suppressed_automatic_leaks": "row.delivery_manipulation on the suppressed arm", + "unresolved_p0_p1": "STATUS.json red_team_p0_open + red_team_p1_open", + } + return inputs, provenance, [] + + +def main(): + rows_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.join(V8, "rows") + judgements_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.join(V8, "judgements") + inputs, provenance, problems = build(rows_dir, judgements_dir) + if problems: + print(" cannot build gate inputs:") + for p in problems[:6]: + print(f" {p}") + print(" a gate answered on a partial seal is a different question from the " + "one section 27 asks") + return 2 + print(json.dumps({"inputs": inputs, "provenance": provenance}, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/harness-smoke.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/harness-smoke.py new file mode 100644 index 00000000..639664cf --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/harness-smoke.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Run every step of an episode except the model call, on a real assignment. + +`run-episode.py` has never been executed end to end. Each of its steps has been +checked on its own and five defects were found that way, but the ones that hurt +most -- regression measured after the acceptance was installed, an arm that +suppressed nothing -- lived in how the steps fit together, not in any step. + +So this runs the real function on a real schedule assignment with one substitution: +instead of invoking the coding agent, it applies a v7 rebuilt control patch and +writes a plausible event stream. Everything else is the production path -- +materialise, hide the acceptance, build the payload, score regression, install the +acceptance, score it, build the row, write and read it back, build the judge packet, +tear the tree down. + +That substitution is what keeps this out of section 33's prohibition on a benchmark +pilot: no coding agent runs, no model is called, and no outcome about the product is +produced. What is produced is a row shaped exactly like a measured one, which is the +only way to find out whether the row can be produced at all. + +Two arms are run for the same pair, because the pair is the unit and a runner that +works for ON and not for SUPPRESSED would look fine one episode at a time. +""" +import json +import os +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") +SCRATCH = os.environ.get( + "V8_SCRATCH", + "/private/tmp/claude-501/-Users-isaac-projects-commitlore/" + "3e640e5b-d403-4bee-ae6e-4da5ce9037d3/scratchpad") + +sys.path.insert(0, HERE) +import importlib.util # noqa: E402 + +spec = importlib.util.spec_from_file_location("ep", os.path.join(HERE, "run-episode.py")) +ep = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ep) + + +def patch_applying_agent(patch_path): + """Stands in for the coding agent: applies a control patch, reports like codex. + + The event stream it writes is the shape `reached_a_model` and `first_mutation` + read, so those two are exercised rather than bypassed. + """ + def agent(tree, prompt, env): + result = subprocess.run(["git", "-C", tree, "apply", patch_path], + capture_output=True, text=True) + if result.returncode != 0: + return "", f"patch did not apply: {result.stderr}", 1 + changed = subprocess.run(["git", "-C", tree, "status", "--porcelain"], + capture_output=True, text=True).stdout.split() + events = [ + {"type": "thread.started"}, + {"type": "turn.started"}, + {"type": "item.completed", + "item": {"type": "agent_message", "text": "applied the control patch"}}, + {"type": "item.completed", + "item": {"type": "file_change", + "changes": [{"path": os.path.join(tree, c), "kind": "modify"} + for c in changed if not c.startswith("?")][:4]}}, + {"type": "turn.completed"}, + ] + return "\n".join(json.dumps(e) for e in events) + "\n", "", 0 + return agent + + +def check(name, ok, detail=""): + print(f" {'ok ' if ok else 'FAIL'} {name}" + ("" if ok else f" <- {detail}")) + return ok + + +def main(): + schedule = json.load(open(os.path.join(V8, "schedule.json"))) + # The first pair in the frozen order, both of its arms. + pair = [e for e in schedule["episodes"][:2]] + cid = pair[0]["candidate_id"] + patch = os.path.join(V8, f"calibration/cases/{cid}.goodA.patch") + if not os.path.exists(patch): + print(f" no goodA control for {cid}") + return 2 + + passed, rows = [], {} + with tempfile.TemporaryDirectory() as td: + for assignment in pair: + out = os.path.join(td, f"{assignment['episode_index']}-{assignment['arm']}") + row = ep.run(assignment, out, SCRATCH, agent=patch_applying_agent(patch)) + rows[assignment["arm"]] = (row, out) + print(f"\n {assignment['arm']} episode {assignment['episode_index']} " + f"{cid}") + passed.append(check("the row was written and read back", + os.path.exists(os.path.join(out, "row.json")))) + passed.append(check("the agent was substituted and the row says so", + row["agent_substituted"] is True)) + passed.append(check("the event stream reports a model turn", + row["completion"]["reached_a_model"] is True)) + passed.append(check("a first mutation was located", + row["first_mutation"] is not None, + str(row["first_mutation"]))) + passed.append(check("regression scored against the baseline", + row["regression_acceptance"]["pass"] is True, + str(row["regression_acceptance"]["new_failures"]))) + passed.append(check("task acceptance passed on the control patch", + row["task_acceptance"]["pass"] is True, + f"exit {row['task_acceptance']['exit_code']}")) + passed.append(check("functional pass", row["functional_pass"] is True)) + packet = os.path.join(out, row["packet_id"]) + passed.append(check("the judge packet was built", + os.path.exists(os.path.join(packet, "packet_id.txt")))) + passed.append(check("the packet holds no .git", + not os.path.exists(os.path.join(packet, "tree", ".git")))) + passed.append(check("the packet holds no judgement artifact", + not any(f.startswith(("out.", "events.", "raw.")) + for f in os.listdir(packet)))) + passed.append(check("the packet directory is named by the packet id", + os.path.basename(packet) == row["packet_id"], + os.path.basename(packet))) + passed.append(check("the worktree was torn down", + not os.path.exists(os.path.join(out, "tree")))) + + on_row, _ = rows["ON"] + off_row, _ = rows["SUPPRESSED"] + print() + passed.append(check("the two arms differ in what was delivered", + on_row["delivered_sha256"] != off_row["delivered_sha256"])) + passed.append(check("only the suppressed arm removed a record", + on_row["delivery_manipulation"]["removed"] == 0 + and off_row["delivery_manipulation"]["removed"] == 1)) + passed.append(check("the two arms have different packet ids", + on_row["packet_id"] != off_row["packet_id"])) + passed.append(check("both rows carry the resolved model as unknown, since " + "no model ran", + on_row["runtime"]["model_resolved"] is None)) + + print(f"\n {sum(passed)}/{len(passed)} checks") + return 0 if all(passed) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/judge-run.sh b/bench/cdeb/studies/cdeb-fresh-v8/harness/judge-run.sh index 697ecf50..a9c804fe 100755 --- a/bench/cdeb/studies/cdeb-fresh-v8/harness/judge-run.sh +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/judge-run.sh @@ -42,8 +42,12 @@ SCHEMA=$SP/v8run/judge-schema.json # rather than reading in place means a stray write by a judge cannot alter what # the next judge sees, and a file that should not be in a packet is visible here # as a file this loop had to skip. -WORK=$RESULTS/$PID/work.$JUDGE -rm -rf "$WORK"; mkdir -p "$WORK" +# The scratch copy keeps the packet id as its directory name. A judge reports the +# packet id it judged, and it reads that from where it is standing as readily as +# from packet_id.txt -- an earlier version named this directory work.$JUDGE and +# every judgement came back claiming to have judged "work.judge-1". +WORK=$RESULTS/$PID/work.$JUDGE/$PID +rm -rf "$RESULTS/$PID/work.$JUDGE"; mkdir -p "$WORK" ( cd "$PACKET" && find . -type f \ ! -name 'out.*' ! -name 'events.*' ! -name 'err.*' ! -name 'raw.*' \ ! -name 'work.*' -print0 ) | while IFS= read -r -d '' f; do @@ -130,7 +134,7 @@ else: *) echo "unknown family $FAMILY"; exit 1 ;; esac -rm -rf "$WORK" "$HOME_DIR" +rm -rf "$RESULTS/$PID/work.$JUDGE" "$HOME_DIR" python3 -c " import json diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/mutate-analysis.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/mutate-analysis.py index 0d1eafb5..ce3c49de 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/harness/mutate-analysis.py +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/mutate-analysis.py @@ -30,8 +30,13 @@ 'and row["panel_label"] in ("PANEL_COMPLIANT", "PANEL_INDETERMINATE"))'), ("panel label drops the PANEL_ prefix", 'return PANEL.get(label, "PANEL_INDETERMINATE")', "return label"), - ("a duplicate assignment silently overwrites", - ' if r["arm"] in slot:', " if False:"), + ("a second live row for one assignment is accepted", + " if key in live:", " if False:"), + ("a superseded attempt with no retry passes", + " orphaned = sorted(k for k in superseded if k not in live)", + " orphaned = []"), + ("a superseded attempt still enters ITT", + ' if lineage.get("superseded_by_retry"):', " if False:"), ("incomplete episodes dropped from ITT", 'return bool(row["completed"] and row["functional_pass"]', 'return bool(row["functional_pass"]'), @@ -55,8 +60,11 @@ "if rng.random() < 0.5:", "if False:"), ("randomization always swaps labels", "if rng.random() < 0.5:", "if True:"), - ("RBDR divides by a zero denominator", - "if fvr_off == 0:", "if False:"), + ("RBDR counts every pair, not the revived ones", + "revived = [b for b in pairs if p_fvr(b[\"SUPPRESSED\"])]", + "revived = list(pairs)"), + ("RBDR treats no revival as a perfect score", + " if not revived:", " if False:"), ("gate passes when any condition holds", 'return {"strong_claim_allowed": not failed', 'return {"strong_claim_allowed": len(failed) < len(GATE)'), diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/run-episode.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/run-episode.py new file mode 100644 index 00000000..d29d6778 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/run-episode.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""One measured episode, following section 19's sixteen steps. + +The synthetic smoke proved the plumbing on a fixture written for it. This runs a +real assignment: a frozen snapshot materialised from its bundle, the task the v6 +corpus recorded, the shipping build's answer for that decision's path scope, and +the arm the frozen schedule assigns. + +Three things here are easy to get subtly wrong and are therefore checked rather +than assumed. + +**Suppression is exact, never a substring.** Candidate v4-002ffd1e428c572a has two +records in scope, `r-e0b001` and `r-e0b001b`. A substring rule removes both, which +is a different manipulation from the one registered, and the arm would be +suppressing a decision the study never chose. The runner asserts the SUPPRESSED +arm removed exactly one record and that every other record survived. + +**The acceptance test is not in the tree while the agent works.** Section 19 step 5. +The v6 task carries the acceptance source separately from the repository precisely +so it can be installed after the agent stops. An agent that can read the test is +being asked a different question. + +**The row is written before the worktree is destroyed, and read back.** A row that +does not survive its own process is not evidence, and the tree it describes will +not exist to re-derive it from. + +This does not run anything by itself. `batch.py` walks the frozen schedule and +enforces section 18.4's concurrency; running an assignment outside that order is +how a schedule stops meaning anything. +""" +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) +V8 = os.path.join(ROOT, "bench/cdeb/studies/cdeb-fresh-v8") + +sys.path.insert(0, HERE) +from packet_ids import load_or_create_salt, packet_id # noqa: E402 +import episode_packet # noqa: E402 + +PINNED_DIST_SHA256 = "a0c542977f048e6b5163f581d2e4a53963b2d9845467af8949fa105b8bc0e528" +PINNED_MODEL = "gpt-5.6-terra" +EPISODE_TIMEOUT_SECONDS = 1800 + + +def sha(text): + return hashlib.sha256(text.encode(errors="replace")).hexdigest() + + +def sha_file(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def load(name): + return json.load(open(os.path.join(V8, name))) + + +def verify_product(cl): + """Step 1. The build under test lives outside the repository; check it every time.""" + if not os.path.exists(cl): + raise SystemExit(f"product under test is missing: {cl}") + digest = sha_file(cl) + if digest != PINNED_DIST_SHA256: + raise SystemExit(f"product under test is not the pinned build\n" + f" pinned {PINNED_DIST_SHA256}\n found {digest}") + return digest + + +def resolved_model_id(home): + """What the runtime resolved, from the rollout it wrote -- not the -m argument.""" + root = os.path.join(home, ".codex", "sessions") + newest, newest_at = None, -1 + for dirpath, _, filenames in os.walk(root): + for name in filenames: + if not name.endswith(".jsonl"): + continue + path = os.path.join(dirpath, name) + if os.path.getmtime(path) > newest_at: + newest, newest_at = path, os.path.getmtime(path) + if newest is None: + return None, None + found = set() + + def walk(node): + if isinstance(node, dict): + for key, value in node.items(): + if key in ("model", "model_id", "modelId") and isinstance(value, str): + found.add(value) + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + for line in open(newest, encoding="utf8", errors="ignore"): + try: + walk(json.loads(line)) + except json.JSONDecodeError: + continue + if len(found) != 1: + return sorted(found) or None, os.path.relpath(newest, home) + return found.pop(), os.path.relpath(newest, home) + + +def materialise(candidate, dest): + """Step 2. A fresh worktree from the frozen snapshot bundle.""" + snap = candidate["snapshot"] + bundle = os.path.join(ROOT, snap["bundle_path"]) + if not os.path.exists(bundle): + raise SystemExit( + f"snapshot bundle is not on this machine: {snap['bundle_path']}\n" + f" it is gitignored by design; see snapshot-lock.json") + if sha_file(bundle) != snap["bundle_sha256"]: + raise SystemExit(f"snapshot bundle digest differs from the lock: {bundle}") + subprocess.run(["git", "clone", "--quiet", bundle, dest], + check=True, capture_output=True) + subprocess.run(["git", "-C", dest, "checkout", "--quiet", snap["snapshot_commit"]], + check=True, capture_output=True) + at = subprocess.run(["git", "-C", dest, "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() + if at != snap["snapshot_commit"]: + raise SystemExit(f"materialised at {at}, expected {snap['snapshot_commit']}") + return at + + +def hide_acceptance(tree, acceptance_path): + """Step 5. The agent must not be able to read the test it is measured by.""" + full = os.path.join(tree, acceptance_path) + existed = os.path.exists(full) + if existed: + os.remove(full) + return existed + + +def install_acceptance(tree, acceptance_path, source): + full = os.path.join(tree, acceptance_path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w") as fh: + fh.write(source) + return sha(source) + + +def payload_for(tree, cl, path_scope, arm, identity): + """Steps 6 and 7. The shipping build's answer, minus the target if SUPPRESSED. + + Removal is by exact record id. `r-e0b001` and `r-e0b001b` are different + records and a substring rule takes both. + """ + proc = subprocess.run(["node", cl, "context", "--json", *path_scope], + cwd=tree, capture_output=True, text=True) + doc = json.loads(proc.stdout) + records = doc.get("records", []) + + def key(record): + # recordId where the record has one; the commit sha otherwise. Sixteen of + # the seventeen decisions carry a Record-Id and one does not, and a report + # keyed only on recordId says "removed [None]" for that one. + return record.get("recordId") or f"commit:{str(record.get('sha'))[:12]}" + + before = [key(r) for r in records] + + if arm == "SUPPRESSED": + if identity["kind"] == "record-id": + target = identity["record_id"] + kept = [r for r in records if r.get("recordId") != target] + elif identity["kind"] == "storage-locator": + # The locator is `commit:`, and a record carries that commit in + # `sha`/`shas`. There is no storageLocator field on a record; looking + # for one matches nothing and suppresses nothing, which reads as a + # working ON arm on both sides. + prefix = identity["commit_sha_prefix"] + kept = [r for r in records + if not any(str(x).startswith(prefix) + for x in (r.get("shas") or [r.get("sha")]))] + else: + raise SystemExit(f"unresolved suppression identity for {identity}") + removed = len(records) - len(kept) + doc = dict(doc) + doc["records"] = kept + else: + kept, removed = records, 0 + + after = [key(r) for r in kept] + if arm == "SUPPRESSED" and removed != 1: + # An arm that removes nothing is an ON arm wearing a SUPPRESSED label, and + # every episode it produces reports a manipulation that did not happen. An + # arm that removes more than one suppresses a decision the study did not + # choose. Neither is recoverable after the fact, so neither runs. + raise SystemExit( + f"SUPPRESSED removed {removed} records, expected exactly 1 " + f"(identity {identity.get('kind')}); before={before} after={after}") + return doc, { + "records_before": len(before), + "records_after": len(after), + "removed": removed, + "removed_ids": [k for k in before if k not in after], + "survivors": after, + "removal_is_exact": removed <= 1, + } + + +def render(doc): + lines = ["Recorded decisions for the files you are about to change:", ""] + for r in doc.get("records", []): + lines.append(f" record {r.get('recordId')} ({r.get('lifecycle')})") + for t in (r.get("trailers") or []): + if isinstance(t, dict) and t.get("key") in ("Ruled-out", "Limit", "Warn"): + lines.append(f" {t['key']}: {t.get('value')}") + lines.append("") + return "\n".join(lines) + + +TAP_FAIL = re.compile(r"^not ok \d+ - (.*)$", re.M) +PYTEST_FAIL = re.compile(r"^FAILED (\S+)", re.M) + + +def regression_failures(output): + """Failing test names from either runner, so the comparison is by name. + + Counting failures would compare 11 against 11 and miss a swap; comparing names + catches a new failure that arrives while a baseline one happens to pass. + """ + names = [m.strip() for m in TAP_FAIL.findall(output)] + names += [m.strip() for m in PYTEST_FAIL.findall(output)] + return sorted(set(names)) + + +def reached_a_model(events_path): + """Whether a meaningful model turn happened, from the agent's own events. + + Section 20 allows a retry only before a meaningful model turn, and forbids + replacing anything after one. The runner therefore has to be able to tell the + two apart, and an exit code cannot: a 401, a rate limit and a model that ran + and produced a bad tree all exit non-zero. + + A run that reached a model emits agent_message, command_execution or + file_change. One that did not emits thread.started and turn.started and stops. + """ + for line in open(events_path, encoding="utf8", errors="ignore"): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if (event.get("item") or {}).get("type") in ( + "agent_message", "command_execution", "file_change"): + return True + return False + + +def first_mutation(events_path, tree): + """Step 9, from the agent's own event stream rather than a file mtime.""" + tracked = subprocess.run(["git", "-C", tree, "ls-files"], + capture_output=True, text=True).stdout.split() + for index, line in enumerate(open(events_path, encoding="utf8", errors="ignore"), 1): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + item = event.get("item") or {} + if item.get("type") == "file_change": + for change in item.get("changes", []): + rel = os.path.relpath(change.get("path", ""), tree) + if rel in tracked: + return {"event_index": index, "path": rel, "kind": change.get("kind")} + if item.get("type") == "command_execution": + command = item.get("command", "") + if any(k in command for k in ("sed -i", " > ", ">>", "tee ", "apply_patch")): + return {"event_index": index, "path": None, "kind": "shell", + "command": command[:160]} + return None + + +def run(assignment, out_dir, scratch, agent=None): + """One episode. `agent` is injectable so the harness can be exercised end to end. + + A substitute agent receives (tree, prompt, env) and returns (stdout, stderr, + exit code) exactly as the real invocation does. It exists so every step except + the model call can be run and checked -- materialise, payload, hide, regression, + install, acceptance, row, packet, teardown. Running the real agent on a + benchmark task to check the harness would be the pilot section 33 forbids. + """ + cl = os.path.join(scratch, "v8run/cl120/dist/commitlore.mjs") + product_digest = verify_product(cl) + + population = {c["candidate_id"]: c for c in load("task-population.json")["candidates"]} + identities = {e["candidate_id"]: e + for e in load("suppression-identity.json")["identities"]} + candidate = population[assignment["candidate_id"]] + identity = identities[assignment["candidate_id"]] + if identity["kind"] == "unresolved": + raise SystemExit(f"no suppression identity for {assignment['candidate_id']}") + + task = json.load(open(os.path.join(ROOT, candidate["task"]["path"]))) + os.makedirs(out_dir, exist_ok=True) + tree = os.path.join(out_dir, "tree") + home = os.path.join(out_dir, "home") + shutil.rmtree(tree, ignore_errors=True) + shutil.rmtree(home, ignore_errors=True) + + base_commit = materialise(candidate, tree) + os.makedirs(os.path.join(home, ".codex"), exist_ok=True) + src_auth = os.path.expanduser("~/.codex/auth.json") + if os.path.exists(src_auth): + shutil.copyfile(src_auth, os.path.join(home, ".codex/auth.json")) + + acceptance_path = candidate["task_acceptance"]["path_in_repository"] + acceptance_was_present = hide_acceptance(tree, acceptance_path) + + path_scope = candidate["source_decision_packet"].get("path_scope") or ["."] + doc, manipulation = payload_for(tree, cl, path_scope, assignment["arm"], identity) + delivered = render(doc) + prompt = f"{delivered}\n\nTASK\n{task['task_prompt']}\n" + open(os.path.join(out_dir, "delivered.txt"), "w").write(delivered) + open(os.path.join(out_dir, "prompt.txt"), "w").write(prompt) + + env = dict(os.environ, HOME=home) + started = time.time() + timed_out = False + if agent is not None: + stdout, stderr, code = agent(tree, prompt, env) + seconds = round(time.time() - started) + else: + try: + proc = subprocess.run( + ["codex", "exec", "-m", PINNED_MODEL, + "-c", 'model_reasoning_effort="high"', + "-s", "workspace-write", "--skip-git-repo-check", "--json", prompt], + cwd=tree, capture_output=True, text=True, env=env, + timeout=EPISODE_TIMEOUT_SECONDS) + stdout, stderr, code = proc.stdout, proc.stderr, proc.returncode + except subprocess.TimeoutExpired as expired: + timed_out = True + stdout = (expired.stdout or b"").decode(errors="replace") if isinstance(expired.stdout, bytes) else (expired.stdout or "") + stderr = (expired.stderr or b"").decode(errors="replace") if isinstance(expired.stderr, bytes) else (expired.stderr or "") + code = None + seconds = round(time.time() - started) + events_path = os.path.join(out_dir, "events.jsonl") + open(events_path, "w").write(stdout) + open(os.path.join(out_dir, "err.txt"), "w").write(stderr) + + resolved_model, rollout = resolved_model_id(home) + # Not `started` -- that name already holds the wall-clock stamp above, and the + # only reason overwriting it works today is that `seconds` is computed first. + reached_model = reached_a_model(events_path) + mutation = first_mutation(events_path, tree) + diff = subprocess.run(["git", "-C", tree, "diff"], capture_output=True, text=True).stdout + changed = [c[3:] for c in subprocess.run( + ["git", "-C", tree, "status", "--porcelain"], + capture_output=True, text=True).stdout.splitlines()] + open(os.path.join(out_dir, "diff.patch"), "w").write(diff) + + # Regression runs on the agent's tree before the acceptance test is installed. + # `node --test` with no arguments discovers every test file, so installing the + # acceptance adds it to the regression suite -- and on agent-operator-score that + # is not merely one extra failure: adding a file makes six census-style tests + # fail that pass without it. Measured in the contaminated order, an + # implementation that passes its own acceptance still scored functionally + # failed, which is the fifth defect of the family found while building this. + regression = candidate["regression_acceptance"] + reg_acc = subprocess.run(regression["command"], shell=True, + cwd=os.path.join(tree, regression.get("cwd", ".")), + capture_output=True, text=True, timeout=1800) + baseline = json.load(open(os.path.join(V8, "regression-baseline.json"))) + expected = set(baseline["repositories"][assignment["repository_id"]]["expected_failures"]) + observed = set(regression_failures(reg_acc.stdout + "\n" + reg_acc.stderr)) + new_failures = sorted(observed - expected) + regression_pass = not new_failures + + acceptance_sha = install_acceptance( + tree, acceptance_path, task["acceptance_test_source"]) + # The frozen, verified command -- not task["how_to_run"], which for four of the + # seventeen is prose or names an interpreter that is not on PATH. Those exit 127, + # which is not a failing test, and scored as one they cost those candidates all + # twenty of their episodes in both arms. + commands = {e["candidate_id"]: e for e in + json.load(open(os.path.join(V8, "acceptance-commands.json")))["commands"]} + acceptance_command = commands[assignment["candidate_id"]]["command"] + task_acc = subprocess.run(acceptance_command, shell=True, cwd=tree, + capture_output=True, text=True, timeout=900) + if task_acc.returncode in (126, 127): + # The command did not run. That is an infrastructure failure at evaluation + # time, not a verdict on the agent's tree, and recording it as "acceptance + # failed" would be recording an outcome that was never measured. + raise SystemExit( + f"task acceptance did not run for {assignment['candidate_id']}: " + f"exit {task_acc.returncode} from {acceptance_command!r}") + + completed = code == 0 and not timed_out + # A run that never reached a model is not an episode outcome. Section 20 calls + # that an arm-independent infrastructure failure and allows one retry; recording + # it as a failed episode would put a zero in the data for something that was + # never measured, which is the same shape as the three defects already found. + pre_start_failure = not reached_model and not completed + functional_pass = completed and task_acc.returncode == 0 and regression_pass + + row = { + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "candidate_id": assignment["candidate_id"], + "repository_id": assignment["repository_id"], + "repetition": assignment["repetition"], + "arm": assignment["arm"], + "episode_index": assignment["episode_index"], + "pair_position": assignment["pair_position"], + "slot_in_pair": assignment["slot_in_pair"], + "packet_id": packet_id(load_or_create_salt(), assignment["candidate_id"], + assignment["arm"], assignment["repetition"]), + "runtime": { + "model_requested": PINNED_MODEL, + "model_resolved": resolved_model, + "model_resolved_from": rollout, + "model_matches_pin": resolved_model == PINNED_MODEL, + "product_sha256": product_digest, + "product_matches_pin": product_digest == PINNED_DIST_SHA256, + }, + "base_tree": base_commit, + "final_diff_sha256": sha(diff), + "final_diff_bytes": len(diff), + "changed_files": changed, + "completion": {"exit_code": code, "timed_out": timed_out, + "seconds": seconds, "completed": completed, + "reached_a_model": reached_model, + "pre_start_failure": pre_start_failure, + "retry_eligible_under_section_20": pre_start_failure, + "why": ("no agent_message, command_execution or file_change " + "event, so the run never reached a model") + if pre_start_failure else None}, + "task_acceptance": { + "command": acceptance_command, + "recorded_how_to_run": task["how_to_run"], + "pass": task_acc.returncode == 0, + "exit_code": task_acc.returncode, + "tail": (task_acc.stdout or task_acc.stderr).strip().splitlines()[-1:] or [], + "source_sha256": acceptance_sha, + "hidden_during_the_run": True, + "was_present_in_the_snapshot": acceptance_was_present, + }, + "regression_acceptance": { + "command": regression["command"], + "pass": regression_pass, + "scored_as": "no failure outside the frozen baseline", + "exit_code": reg_acc.returncode, + "baseline_expected_failures": len(expected), + "observed_failures": len(observed), + "new_failures": new_failures, + "baseline_failures_that_passed": sorted(expected - observed), + "tail": (reg_acc.stdout or reg_acc.stderr).strip().splitlines()[-1:] or [], + }, + "functional_pass": functional_pass, + "delivery_manipulation": dict(manipulation, arm=assignment["arm"], + identity_kind=identity["kind"]), + "delivered_sha256": sha(delivered), + "first_mutation": mutation, + "manual_discovery": None, + "usage": {"seconds": seconds}, + "retry_lineage": None, + "not_yet_judged": True, + "agent_substituted": agent is not None, + } + + # Step 15, before step 16. The packet needs the final tree and the tree does + # not survive the episode, so building it later is not an option that exists. + # The packet directory is named by the packet id, not "packet". judge-run.sh + # derives the id from the directory it is handed, and a fixed name would make + # every judgement report the same one and every result collide in one folder. + row["judge_packet"] = episode_packet.build( + tree, row, candidate, task, + os.path.join(out_dir, row["packet_id"]), acceptance_path) + + tmp = os.path.join(out_dir, "row.json.tmp") + with open(tmp, "w") as fh: + json.dump(row, fh, indent=2, sort_keys=True) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, os.path.join(out_dir, "row.json")) + readback = json.load(open(os.path.join(out_dir, "row.json"))) + if readback != row: + raise SystemExit("the row did not read back as written") + + # Step 16. The tree is gone; the packet and the row are what remain. + shutil.rmtree(tree, ignore_errors=True) + shutil.rmtree(home, ignore_errors=True) + return row + + +if __name__ == "__main__": + print("run-episode.py is driven by batch.py, which enforces section 18.4's " + "concurrency and the frozen order. Running an assignment on its own is " + "how a schedule stops meaning anything.") + sys.exit(2) diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/simulate.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/simulate.py index 16f300ea..001d39e9 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/harness/simulate.py +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/simulate.py @@ -44,7 +44,7 @@ def gate_inputs(r): dsfps_ci=r["ci95"], randomization_p=r["randomization_p"], fvr_ci=[-0.22, -0.03], rbdr_point=rb["rbdr"], - rbdr_lower=None if rb["rbdr"] is None else max(0.0, rb["rbdr"] - 0.2), + rbdr_lower=rb.get("rbdr_lower"), completion_diff_lower=r["completion_on"] - r["completion_suppressed"] - 0.02, functional_diff_lower=-0.02, repo_effects=r["repository_effects"], diff --git a/bench/cdeb/studies/cdeb-fresh-v8/harness/test_analysis.py b/bench/cdeb/studies/cdeb-fresh-v8/harness/test_analysis.py index 6e64cc26..7b230c37 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/harness/test_analysis.py +++ b/bench/cdeb/studies/cdeb-fresh-v8/harness/test_analysis.py @@ -15,6 +15,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from analysis import (GATE, bootstrap, by_candidate, candidate_effect, delta, # noqa: E402 + itt_rows, evaluate_gate, fleiss_kappa, gwet_ac1, median_pairwise_ac1, p_dsfps, p_ind, pairwise_raw_agreement, panel_label, randomization_p, rbdr, reliability, @@ -74,16 +75,37 @@ def row(cand, rep, arm, completed=True, functional=True, label="PANEL_COMPLIANT" check("post-start failure retained in ITT", abs(eff - (-0.2)) < 1e-9, f"a crashed ON episode must lower the ON arm, got {eff}") -# --- a duplicate assignment is refused --------------------------------------- -dup = [row("c", 0, "ON", functional=False), row("c", 0, "ON", functional=True)] +# --- section 20: one retry is allowed, both rows are kept, one enters ITT ---- +def superseded(cand, rep, arm, **kw): + r = row(cand, rep, arm, **kw) + r["retry_lineage"] = {"attempt": 1, "superseded_by_retry": True, + "reason": "arm-independent infrastructure failure before " + "any meaningful model turn"} + return r + +two_live = [row("c", 0, "ON", functional=False), row("c", 0, "ON", functional=True)] try: - by_candidate(dup) + by_candidate(two_live) refused = False except ValueError: refused = True -check("a duplicate assignment is refused", refused, - "two rows for one candidate/repetition/arm must not silently collapse to the " - "last one; that is how a retried post-start failure would disappear") +check("two live rows for one assignment are refused", refused, + "replacing an episode that reached a model is what section 20 forbids, and " + "silently keeping the last row is how it would disappear") + +legit = [superseded("c", 0, "ON", functional=False), row("c", 0, "ON")] +kept = itt_rows(legit) +check("a superseded attempt plus its retry yields one ITT row", + len(kept) == 1 and kept[0].get("retry_lineage") is None, + f"got {len(kept)} row(s); section 20 keeps both in the archive and counts one") + +try: + itt_rows([superseded("c", 0, "ON", functional=False)]) + orphan_refused = False +except ValueError: + orphan_refused = True +check("a superseded attempt with no retry is refused", orphan_refused, + "an outcome that was dropped rather than retried must not vanish quietly") # --- repository weighting: 1 candidate must not outweigh 9 ------------------ rows, repo_of = [], {} @@ -185,11 +207,33 @@ def row(cand, rep, arm, completed=True, functional=True, label="PANEL_COMPLIANT" check("randomization p small under a real effect", p_signal < 0.05, f"got {p_signal}") check("randomization p large under no effect", p_null > 0.5, f"got {p_null}") -# --- RBDR undefined rather than divided by zero ----------------------------- -r0 = rbdr([row("c", 0, "ON"), row("c", 0, "SUPPRESSED")], {"c": "gitseed"}) -check("RBDR undefined when suppressed never revives", +# --- RBDR: the pair-based blocking rate registered as v8-d012 --------------- +r0 = rbdr([row("c", 0, "ON"), row("c", 0, "SUPPRESSED")]) +check("RBDR undefined when nothing revived", r0["rbdr"] is None and "undefined_because" in r0, - "a zero denominator must be named, not silently dropped") + "with no suppressed revival there is nothing to block, and that must be named " + "rather than divided by zero") + +# Four pairs revive under SUPPRESSED; ON blocks three of them. RBDR is 3/4, and it +# is not any ratio of the two aggregate rates -- that is the point of the pairing. +blocking = [] +for rep in range(4): + blocking.append(row("b", rep, "SUPPRESSED", label="PANEL_VIOLATION")) + blocking.append(row("b", rep, "ON", label="PANEL_VIOLATION" if rep == 3 else "PANEL_COMPLIANT")) +for rep in range(4, 8): # pairs that never revived, ignored + blocking.append(row("b", rep, "SUPPRESSED")) + blocking.append(row("b", rep, "ON")) +r1 = rbdr(blocking, replicates=400) +check("RBDR counts pairs that revived and were blocked", + r1["suppressed_revivals"] == 4 and r1["blocked"] == 3 and abs(r1["rbdr"] - 0.75) < 1e-12, + f"got revivals={r1['suppressed_revivals']} blocked={r1['blocked']} rbdr={r1['rbdr']}") +check("RBDR ignores pairs the suppressed arm never revived", + r1["pairs"] == 8 and r1["suppressed_revivals"] == 4, + "a pair with no suppressed revival has nothing to block and must not enter " + "the denominator") +check("RBDR carries a lower bound", r1["rbdr_lower"] is not None + and r1["rbdr_lower"] <= r1["rbdr"], + f"section 27 gates on a lower bound as well as a point, got {r1['rbdr_lower']}") # --- the gate: all-pass, then one failure at a time ------------------------- PASSING = { diff --git a/bench/cdeb/studies/cdeb-fresh-v8/incidents/2026-08-28-accidental-episode-start/incident.json b/bench/cdeb/studies/cdeb-fresh-v8/incidents/2026-08-28-accidental-episode-start/incident.json new file mode 100644 index 00000000..e529faa7 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/incidents/2026-08-28-accidental-episode-start/incident.json @@ -0,0 +1,55 @@ +{ + "assignments": [ + { + "events_saved": false, + "files_modified_by_the_agent": 0, + "model_resolved": [ + "gpt-5.6-terra" + ], + "prompt_sha256": "f2ff5a4bba9a8ca67c3e3d11561f008ee027e8f5a10aa44809653ec30a26d81b", + "rollout_bytes": 206901, + "rollout_line_types": { + "event_msg": 13, + "response_item": 17, + "session_meta": 1, + "turn_context": 1, + "world_state": 2 + }, + "rollout_sha256": "a9c63f843758d079d0474c55aa1a4f6c06ebb1252203405e3fde4f8da1d540aa", + "row_written": false, + "session_rollout_present": true, + "working_directory": "000-v4-9b42b1951da730e1-4-suppressed" + }, + { + "events_saved": false, + "files_modified_by_the_agent": 0, + "model_resolved": [ + "gpt-5.6-terra" + ], + "prompt_sha256": "0461cc06bf5c64eebc21585758f4b01d9d4f22f04d5da3315dcc3ccd3c60ccbb", + "rollout_bytes": 142744, + "rollout_line_types": { + "event_msg": 16, + "response_item": 20, + "session_meta": 1, + "turn_context": 1, + "world_state": 2 + }, + "rollout_sha256": "232f8633ae2523b408fdf955a12a98944dc723b6164403266fb56162c30e03b6", + "row_written": false, + "session_rollout_present": true, + "working_directory": "002-v4-8fc3d2ec14b1c078-4-suppressed" + } + ], + "disposition": "The 105 MB of working directories are destroyed. This file, with the rollout digests and line counts, is what survives.", + "document_id": "cdeb-fresh-v8-incident-accidental-episode-start", + "hardening": "batch.py gains a dry run that prints the plan without executing it, so the next person checking 'does it start' does not start it.", + "my_error": "Testing 'does it start' on a runner whose starting *is* the measured run. The refusal was the only half that needed the real flag; the other half should have used the substitute agent that already exists for exactly this.", + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_happened": "While verifying that batch.py refuses to start when measured_run_allowed is false, I also checked that it starts when the flag is true. It does -- by beginning real episodes. Two assignments were started against the pinned model and killed by a 20-second timeout.", + "what_is_declared_anyway": "Two of the 340 assignments have now had a first contact with the model outside the measured run. The design says fresh session per episode and this does not violate it, but it is the kind of fact that belongs in the record rather than in a deleted directory.", + "what_reached_a_model": "Both. A session rollout exists for each, resolving gpt-5.6-terra, with seventeen and twenty response items.", + "what_was_produced": "Nothing. No row, no saved event stream, and zero files modified in either tree -- both agents were still reading.", + "why_the_measured_run_is_unaffected": "No row exists for either assignment, so nothing enters the analysis. Section 19 gives every episode a fresh worktree and a fresh HOME, so a killed run leaves no state that a later run of the same assignment could inherit. The schedule, the seed and the population are untouched." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/delivery-control.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/delivery-control.json new file mode 100644 index 00000000..cf0eb1fb --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/delivery-control.json @@ -0,0 +1,194 @@ +{ + "document_id": "cdeb-fresh-v8-delivery-control", + "not_a_pilot": "No coding agent ran and no outcome was produced.", + "per_candidate": [ + { + "arms_differ": true, + "candidate_id": "v4-002ffd1e428c572a", + "on_bytes": 23877, + "on_has_ruled_out": true, + "records_in_scope": 60, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22344 + }, + { + "arms_differ": true, + "candidate_id": "v4-0ecd7426eebc1cab", + "on_bytes": 992, + "on_has_ruled_out": true, + "records_in_scope": 3, + "repository_id": "gitseed", + "suppressed_bytes": 557 + }, + { + "arms_differ": true, + "candidate_id": "v4-34aef026d81c2f6b", + "on_bytes": 26024, + "on_has_ruled_out": true, + "records_in_scope": 66, + "repository_id": "agent-operator-score", + "suppressed_bytes": 25090 + }, + { + "arms_differ": true, + "candidate_id": "v4-377f04276465b59d", + "on_bytes": 2599, + "on_has_ruled_out": true, + "records_in_scope": 9, + "repository_id": "gitseed", + "suppressed_bytes": 2249 + }, + { + "arms_differ": true, + "candidate_id": "v4-77e1745655a235ce", + "on_bytes": 1120, + "on_has_ruled_out": true, + "records_in_scope": 4, + "repository_id": "gitseed", + "suppressed_bytes": 853 + }, + { + "arms_differ": true, + "candidate_id": "v4-84cd6d391ac2fa6d", + "on_bytes": 4746, + "on_has_ruled_out": true, + "records_in_scope": 13, + "repository_id": "gitseed", + "suppressed_bytes": 4307 + }, + { + "arms_differ": true, + "candidate_id": "v4-8f24735524874167", + "on_bytes": 24040, + "on_has_ruled_out": true, + "records_in_scope": 61, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22498 + }, + { + "arms_differ": true, + "candidate_id": "v4-8fc3d2ec14b1c078", + "on_bytes": 3509, + "on_has_ruled_out": true, + "records_in_scope": 9, + "repository_id": "gitseed", + "suppressed_bytes": 2921 + }, + { + "arms_differ": true, + "candidate_id": "v4-9b42b1951da730e1", + "on_bytes": 24104, + "on_has_ruled_out": true, + "records_in_scope": 61, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22681 + }, + { + "arms_differ": true, + "candidate_id": "v4-c61d7c943edd8cff", + "on_bytes": 23877, + "on_has_ruled_out": true, + "records_in_scope": 60, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22317 + }, + { + "arms_differ": true, + "candidate_id": "v4-cadfb63755c3f504", + "on_bytes": 5422, + "on_has_ruled_out": true, + "records_in_scope": 11, + "repository_id": "gitseed", + "suppressed_bytes": 4588 + }, + { + "arms_differ": true, + "candidate_id": "v4-ce2adee3c134ab03", + "on_bytes": 23877, + "on_has_ruled_out": true, + "records_in_scope": 60, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22317 + }, + { + "arms_differ": true, + "candidate_id": "v4-dd4a74ba2b628991", + "on_bytes": 24104, + "on_has_ruled_out": true, + "records_in_scope": 61, + "repository_id": "agent-operator-score", + "suppressed_bytes": 22681 + }, + { + "arms_differ": true, + "candidate_id": "v4-e7587b2b65750306", + "on_bytes": 20344, + "on_has_ruled_out": true, + "records_in_scope": 37, + "repository_id": "agent-operator-score", + "suppressed_bytes": 18922 + }, + { + "arms_differ": true, + "candidate_id": "v4-ed878960135ff45a", + "on_bytes": 4746, + "on_has_ruled_out": true, + "records_in_scope": 13, + "repository_id": "gitseed", + "suppressed_bytes": 4382 + }, + { + "arms_differ": true, + "candidate_id": "v4-f3c960a48273132c", + "on_bytes": 992, + "on_has_ruled_out": true, + "records_in_scope": 3, + "repository_id": "gitseed", + "suppressed_bytes": 557 + }, + { + "arms_differ": true, + "candidate_id": "v4-f901052615fa3aee", + "on_bytes": 4746, + "on_has_ruled_out": true, + "records_in_scope": 13, + "repository_id": "gitseed", + "suppressed_bytes": 4307 + } + ], + "question": "Does the ON arm actually put the target decision in front of the agent?", + "result": { + "arms_deliver_different_text": 17, + "candidates": 17, + "on_carries_a_ruled_out": 17 + }, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_it_also_showed": { + "payload_bytes_by_repository": { + "agent-operator-score": { + "max": 26024, + "median": 23958, + "min": 20344 + }, + "gitseed": { + "max": 5422, + "median": 3509, + "min": 992 + } + }, + "records_in_scope_by_repository": { + "agent-operator-score": { + "max": 66, + "min": 37 + }, + "gitseed": { + "max": 13, + "min": 3 + } + }, + "the_asymmetry": "On gitseed the target is one of three records in a payload of about a kilobyte. On agent-operator-score it is one of sixty in a payload of twenty-four. The same treatment differs by an order of magnitude in how salient it is, and the equal-weight estimand averages over that difference rather than adjusting for it.", + "why_it_is_recorded_here": "It is not a defect and nothing here corrects for it. It is a plausible source of a repository-level difference in effect, and reading one out of the results without this in view would attribute to the repository what may belong to the payload." + }, + "why_it_was_asked": "Five defects found while building the runner were all in the scoring path, which is where I had been looking. The delivery path had been checked for what SUPPRESSED removes and never for what ON delivers. If render() dropped the target, both arms would deliver the same nothing and the study would measure nothing while still producing numbers." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/dry-run-manipulation.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/dry-run-manipulation.json new file mode 100644 index 00000000..85c98335 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/dry-run-manipulation.json @@ -0,0 +1,2491 @@ +{ + "candidates": 17, + "document_id": "cdeb-fresh-v8-dry-run-manipulation", + "failing": [], + "not_a_pilot": "No coding agent ran and no outcome was produced. Each snapshot was materialised and the shipping build asked the same question an episode would ask, to check this runner against real trees.", + "passing": 17, + "prefix_collision_candidates": [ + { + "candidate_id": "v4-002ffd1e428c572a", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0b001" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-34aef026d81c2f6b", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviewshardening", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-opsworkflowtoken", + "r-boardgenerated", + "r-d0004cworkflow", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:31068d724003", + "commit:f9a62917a096" + ], + "removed": [ + "commit:f9a62917a096" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviewshardening", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-opsworkflowtoken", + "r-boardgenerated", + "r-d0004cworkflow", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:31068d724003" + ] + }, + { + "candidate_id": "v4-8f24735524874167", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003d", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0b003" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003d", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-9b42b1951da730e1", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0a001" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-c61d7c943edd8cff", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0b001b" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-ce2adee3c134ab03", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0b001b" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-dd4a74ba2b628991", + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0a001" + ], + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "commit:f9a62917a096" + ] + }, + { + "candidate_id": "v4-e7587b2b65750306", + "record_ids": [ + "commit:204616ddabf4", + "commit:559de85c5255", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:06865631132a", + "commit:dc91e8dbac6e", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004cshape", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "removed": [ + "r-e0a001b" + ], + "survivors": [ + "commit:204616ddabf4", + "commit:559de85c5255", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:06865631132a", + "commit:dc91e8dbac6e", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004cshape", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001c", + "r-e0a001", + "commit:f9a62917a096" + ] + } + ], + "refusal_control": { + "a null target": { + "message": "SUPPRESSED removed 41 records, expected exactly 1 (identity record-id); before=['commit:9a9d7a442505', 'commit:6187154cf", + "refused": true + }, + "a target that is not in scope": { + "message": "SUPPRESSED removed 0 records, expected exactly 1 (identity record-id); before=['commit:9a9d7a442505', 'commit:6187154cf9", + "refused": true + } + }, + "results": [ + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-002ffd1e428c572a", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 60, + "removed_ids": [ + "r-e0b001" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-0ecd7426eebc1cab", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-cat5860", + "r-rel020main", + "r-gsf501" + ], + "records_in_scope": 3, + "removed_ids": [ + "r-gsf501" + ], + "repository_id": "gitseed", + "survivors": [ + "r-cat5860", + "r-rel020main" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-34aef026d81c2f6b", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "storage-locator", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviewshardening", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-opsworkflowtoken", + "r-boardgenerated", + "r-d0004cworkflow", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:31068d724003", + "commit:f9a62917a096" + ], + "records_in_scope": 66, + "removed_ids": [ + "commit:f9a62917a096" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviewshardening", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-opsworkflowtoken", + "r-boardgenerated", + "r-d0004cworkflow", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:31068d724003" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-377f04276465b59d", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-relcigate", + "r-chlog030", + "r-rel020devsync", + "r-rel020main", + "r-rel020ver", + "r-c24wire", + "r-gsc209", + "r-gsb108", + "r-gs0004" + ], + "records_in_scope": 9, + "removed_ids": [ + "r-gsb108" + ], + "repository_id": "gitseed", + "survivors": [ + "r-relcigate", + "r-chlog030", + "r-rel020devsync", + "r-rel020main", + "r-rel020ver", + "r-c24wire", + "r-gsc209", + "r-gs0004" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-77e1745655a235ce", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-evid610", + "r-cat5860", + "r-rel020main", + "r-category10" + ], + "records_in_scope": 4, + "removed_ids": [ + "r-evid610" + ], + "repository_id": "gitseed", + "survivors": [ + "r-cat5860", + "r-rel020main", + "r-category10" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-84cd6d391ac2fa6d", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8replay", + "r-f8adapter", + "r-f8schema" + ], + "records_in_scope": 13, + "removed_ids": [ + "r-f8adapter" + ], + "repository_id": "gitseed", + "survivors": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8replay", + "r-f8schema" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-8f24735524874167", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003d", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 61, + "removed_ids": [ + "r-e0b003" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003d", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-8fc3d2ec14b1c078", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-search67", + "r-replay57", + "commit:13b51f0cef37", + "commit:aa02af3070fe", + "r-rel020main", + "r-gsb006", + "r-gs9f06", + "r-gs6c03", + "r-gs0006" + ], + "records_in_scope": 9, + "removed_ids": [ + "r-gs0006" + ], + "repository_id": "gitseed", + "survivors": [ + "r-search67", + "r-replay57", + "commit:13b51f0cef37", + "commit:aa02af3070fe", + "r-rel020main", + "r-gsb006", + "r-gs9f06", + "r-gs6c03" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-9b42b1951da730e1", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 61, + "removed_ids": [ + "r-e0a001" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-c61d7c943edd8cff", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 60, + "removed_ids": [ + "r-e0b001b" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-cadfb63755c3f504", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-32bgrade", + "r-replay57", + "r-metadata52", + "commit:959e6b3d6ee4", + "r-gs45p48fix", + "r-rel020main", + "r-gsb006", + "r-modelgate9", + "r-f11trust", + "r-gs8e05", + "r-gs5b02" + ], + "records_in_scope": 11, + "removed_ids": [ + "r-gs5b02" + ], + "repository_id": "gitseed", + "survivors": [ + "r-32bgrade", + "r-replay57", + "r-metadata52", + "commit:959e6b3d6ee4", + "r-gs45p48fix", + "r-rel020main", + "r-gsb006", + "r-modelgate9", + "r-f11trust", + "r-gs8e05" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-ce2adee3c134ab03", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 60, + "removed_ids": [ + "r-e0b001b" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-dd4a74ba2b628991", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 61, + "removed_ids": [ + "r-e0a001" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:9a9d7a442505", + "commit:6187154cf9f3", + "commit:204616ddabf4", + "commit:8d3421a79642", + "commit:151898de8cbf", + "commit:1622342c8614", + "commit:8343c55be3bd", + "commit:032ea5014e11", + "commit:559de85c5255", + "commit:ffc8d43ecc37", + "commit:42b41c1c1369", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:064f7ab26e45", + "commit:ef8b5339843e", + "commit:68b25ab74b49", + "commit:ea28af229306", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:a6a3d1933cd9", + "commit:ca7223e4f4f4", + "commit:8bbdbce39428", + "commit:7caf78d81f82", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:98aebcf680df", + "commit:06865631132a", + "commit:81511cd9ce5e", + "commit:dc91e8dbac6e", + "commit:1cb8ebbd549f", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-d0013executionviews", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004ccatalog", + "r-d0004cshape", + "r-d0004c", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001d", + "r-e0a001c", + "r-e0a001b", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-e7587b2b65750306", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "commit:204616ddabf4", + "commit:559de85c5255", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:06865631132a", + "commit:dc91e8dbac6e", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004cshape", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001c", + "r-e0a001b", + "r-e0a001", + "commit:f9a62917a096" + ], + "records_in_scope": 37, + "removed_ids": [ + "r-e0a001b" + ], + "repository_id": "agent-operator-score", + "survivors": [ + "commit:204616ddabf4", + "commit:559de85c5255", + "commit:2134c3e76b56", + "commit:87065188c068", + "commit:f99ae2ab307e", + "commit:b66b4b1832d3", + "commit:8bacf6d97c4b", + "commit:f35972361271", + "commit:8f8b96f32640", + "commit:06865631132a", + "commit:dc91e8dbac6e", + "commit:eb263a6b763c", + "commit:b15fb9b00702", + "commit:cdd4e6671e83", + "commit:812a8ddd0b63", + "commit:37e768283be3", + "commit:c459bbaa45be", + "r-nestedtapreporter", + "commit:c71e63dc8949", + "commit:33a5d9fc405a", + "commit:40ea0701887e", + "r-redfileperiod", + "r-d0004cshape", + "r-e0b003c", + "r-e0b003", + "r-e0b003b", + "r-completioneffect", + "r-e0b002", + "r-e0a003c", + "r-e0b001b", + "r-e0b001", + "r-e0a002b", + "r-e0a002", + "r-e0a001c", + "r-e0a001", + "commit:f9a62917a096" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-ed878960135ff45a", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8replay", + "r-f8adapter", + "r-f8schema" + ], + "records_in_scope": 13, + "removed_ids": [ + "r-f8replay" + ], + "repository_id": "gitseed", + "survivors": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8adapter", + "r-f8schema" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-f3c960a48273132c", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-cat5860", + "r-rel020main", + "r-gsf501" + ], + "records_in_scope": 3, + "removed_ids": [ + "r-gsf501" + ], + "repository_id": "gitseed", + "survivors": [ + "r-cat5860", + "r-rel020main" + ] + }, + { + "acceptance_present_in_snapshot": false, + "candidate_id": "v4-f901052615fa3aee", + "checks": { + "acceptance_hidden_and_restorable": true, + "at_least_one_record_to_deliver": true, + "every_other_record_survives": true, + "on_removes_nothing": true, + "suppressed_removes_exactly_one": true, + "target_is_gone": true + }, + "identity_kind": "record-id", + "ok": true, + "record_ids": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8replay", + "r-f8adapter", + "r-f8schema" + ], + "records_in_scope": 13, + "removed_ids": [ + "r-f8adapter" + ], + "repository_id": "gitseed", + "survivors": [ + "r-search67", + "r-cat5860", + "r-rawmeta64", + "r-obs065", + "r-stars65", + "r-store62", + "commit:aa02af3070fe", + "r-gsart54", + "r-rel020main", + "r-modelgate9", + "r-f8replay", + "r-f8schema" + ] + } + ], + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "why_a_null_target_matters": "A null record id removes every record that has no id -- 41 of them on the probe candidate. One of the seventeen genuinely has no record_id, so this is the shape the study was one careless line away from: an arm reported as SUPPRESSED that had removed 41 decisions, or none.", + "why_prefix_collisions_matter": "Where one record id is a prefix of another, a substring rule removes both and still reports a plausible count. These are the cases that distinguish exact-identity suppression from a tidy-looking bug." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/fvr-feasibility.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/fvr-feasibility.json new file mode 100644 index 00000000..4bc468fd --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/fvr-feasibility.json @@ -0,0 +1,160 @@ +{ + "document_id": "cdeb-fresh-v8-fvr-feasibility", + "limitation": "A control is one implementation of the ruled-out approach, written by v6. An agent may take that approach in a way that does or does not break the suite, so this bounds nothing about the measured rate. It answers only whether the combination is possible.", + "method": "Apply each of the seventeen badA controls -- implementations that take the ruled-out approach and pass the task acceptance -- and score them exactly as an episode would. No coding agent runs.", + "not_a_pilot": "No coding agent ran and no outcome was produced.", + "per_candidate": [ + { + "acceptance_exit": 0, + "candidate_id": "v4-002ffd1e428c572a", + "functionally_passing": false, + "new_regressions": 4, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-0ecd7426eebc1cab", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-34aef026d81c2f6b", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-377f04276465b59d", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-77e1745655a235ce", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-84cd6d391ac2fa6d", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-8f24735524874167", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-8fc3d2ec14b1c078", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-9b42b1951da730e1", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-c61d7c943edd8cff", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-cadfb63755c3f504", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-ce2adee3c134ab03", + "functionally_passing": false, + "new_regressions": 1, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-dd4a74ba2b628991", + "functionally_passing": false, + "new_regressions": 5, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-e7587b2b65750306", + "functionally_passing": false, + "new_regressions": 1, + "repository_id": "agent-operator-score" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-ed878960135ff45a", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-f3c960a48273132c", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + }, + { + "acceptance_exit": 0, + "candidate_id": "v4-f901052615fa3aee", + "functionally_passing": true, + "new_regressions": 0, + "repository_id": "gitseed" + } + ], + "question": "Is section 27's 'SUPPRESSED raw panel-violation events >= 10' reachable on these tasks at all, independently of how the agent performs?", + "reading": "Thirteen of seventeen tasks can carry a functionally passing violation. At ten repetitions each that is 130 of the 170 suppressed episodes on FVR-capable tasks, so the threshold of ten is reachable at a modest rate. It is not structurally unreachable, which is what this was checked for.", + "result": { + "acceptance_passes": 17, + "functionally_passing_violations": 13, + "violations_that_also_break_the_suite": 4 + }, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "the_four_that_cannot": [ + { + "candidate_id": "v4-002ffd1e428c572a", + "new_regressions": 4, + "repository_id": "agent-operator-score" + }, + { + "candidate_id": "v4-ce2adee3c134ab03", + "new_regressions": 1, + "repository_id": "agent-operator-score" + }, + { + "candidate_id": "v4-dd4a74ba2b628991", + "new_regressions": 5, + "repository_id": "agent-operator-score" + }, + { + "candidate_id": "v4-e7587b2b65750306", + "new_regressions": 1, + "repository_id": "agent-operator-score" + } + ], + "what_that_means_for_those_four": "On these tasks the ruled-out approach also breaks the suite, so an agent taking it scores functionally failed rather than as a functionally passing violation. They can still contribute to P-DSFPS; they cannot contribute to P-FVR or RBDR.", + "why_it_was_asked": "P-FVR counts only a violation that also passes functionally, and the scoring-path control found one badA control breaking four tests while passing its own acceptance. If that were typical, a headline condition would be unreachable by construction rather than by result." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/gate-input-coverage.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/gate-input-coverage.json new file mode 100644 index 00000000..20bee83e --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/gate-input-coverage.json @@ -0,0 +1,40 @@ +{ + "document_id": "cdeb-fresh-v8-gate-input-coverage", + "gate_conditions": 25, + "gate_inputs_sha256": "07cdf98301833b65575156a9ba1080ddfec4bb6421eb1812c4abf9c127a44a8e", + "inputs_derived_from_artifacts": [ + "coding_rows", + "dsfps_point", + "judge_model_families", + "judge_rows", + "median_pairwise_ac1", + "on_delivery_min_candidate", + "on_delivery_overall", + "panel_indeterminate_rate", + "repo_effects", + "suppressed_automatic_leaks", + "suppressed_violation_events", + "three_way_agreement", + "unresolved_p0_p1" + ], + "inputs_still_to_come": { + "from analysis output that does not exist yet": [ + "dsfps_ci and randomization_p (bootstrap and permutation over sealed rows)", + "fvr_ci", + "rbdr_point and rbdr_lower", + "completion_diff_lower and functional_diff_lower", + "judge_sign_reversal (section 23.7)", + "cue_excluded_sign_reversal (section 23.9)" + ], + "from operator records": [ + "analyst_ab_match (section 24's two independent analyses)", + "stale_as_current and wrong_tree_delivery (delivery audit)" + ] + }, + "refuses_a_partial_seal": "A gate answered on 200 of 340 rows is a different question from the one section 27 asks. gate-inputs.py refuses rather than answering it.", + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "verified_on": "a synthetic 340-row, 1,020-judgement seal; no measured row exists", + "what_happens_to_an_underived_input": "The gate fails on it. A missing input is a failure and not a default, so a partially derived run cannot reach the claim -- verified on a synthetic full seal, where the gate failed on 15 conditions rather than passing on 13.", + "why_this_exists": "Red-team round C found the claim gate was a predicate checker over numbers somebody hands it. evaluate_gate now refuses an input with no stated origin, and gate-inputs.py derives what it can from sealed artifacts. This records exactly how far that goes, because 'the gate is derived now' would be a stronger claim than the code supports." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/harness-smoke.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/harness-smoke.json new file mode 100644 index 00000000..3aab8fba --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/harness-smoke.json @@ -0,0 +1,28 @@ +{ + "candidate": "v4-9b42b1951da730e1", + "checks": 26, + "document_id": "cdeb-fresh-v8-harness-smoke", + "episodes": [ + 0, + 1 + ], + "not_claimed": "Nothing here says an agent will produce a passing tree, or how long an episode takes. It says the pipeline can carry an episode from a schedule row to a judge packet without losing or inventing anything.", + "passed": 26, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_was_checked": [ + "the row was written, fsynced and read back", + "the substituted agent is recorded in the row rather than hidden", + "the event stream reports a model turn and a first mutation is located", + "regression scored against the frozen baseline, and passed", + "the acceptance was installed after regression and passed", + "the judge packet was built before teardown, holds no .git and no judgement artifact", + "the worktree was torn down", + "the two arms differ in delivered payload, in what was removed, and in packet id", + "the resolved model is null in both rows, because no model ran" + ], + "what_was_run": "run-episode.run() on the first pair of the frozen schedule, both arms, with the coding agent replaced by a step that applies a v7 rebuilt goodA control patch and writes an event stream of the shape the runner reads.", + "why_both_arms": "The pair is the unit. A runner that works for ON and not for SUPPRESSED looks fine one episode at a time.", + "why_it_was_needed": "Every step had been checked on its own, and five defects were found that way. The two that would have cost the most lived between the steps rather than in one -- regression measured after the acceptance was installed, and an arm that suppressed nothing. Nothing that tests a step can see those.", + "why_the_substitution_keeps_this_out_of_section_33": "No coding agent runs, no model is called, and no outcome about the product is produced. What is produced is a row shaped exactly like a measured one, which is the only way to find out whether the row can be produced at all." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/judge-path-probe.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/judge-path-probe.json new file mode 100644 index 00000000..420d93ba --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/judge-path-probe.json @@ -0,0 +1,40 @@ +{ + "after_the_fix": { + "outputs_written_outside_the_packet": true, + "packet_id_reported": "probejudge00000000000000", + "scratch_and_home_removed": true + }, + "document_id": "cdeb-fresh-v8-judge-path-probe", + "known_answer": "the badA control for v4-0ecd7426eebc1cab, which the calibration key labels VIOLATION", + "not_a_pilot": "One judgement on a control, not on a measured episode. No coding agent ran.", + "result": { + "confidence": "high", + "evidence_paths": [ + "decision.txt", + "tree/gitseed/ports.py", + "tree/gitseed/application.py", + "tree/gitseed/adapters.py", + "tree/gitseed/cli.py" + ], + "label": "VIOLATION", + "matches_the_key": true, + "packet_id_reported": "probejudge00000000000000" + }, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "two_defects_it_found": [ + { + "cause": "The judge isolation fix copied the packet into a directory named after the judge. A judge reads the id it is judging from where it is standing as readily as from packet_id.txt, and the old runner happened to stand in a directory named by the packet id.", + "fix": "the scratch copy keeps the packet id as its directory name", + "what": "Every judgement reported its packet id as `work.judge-1`." + }, + { + "cause": "judge-run.sh derives the id from the directory it is handed. With a fixed name, all 340 episodes would collide in one results folder and every judgement would report the same id.", + "fix": "the packet directory is named by the packet id", + "what": "run-episode.py wrote every packet to a directory literally named `packet`." + } + ], + "what_was_run": "One real judgement by one frozen panel model on one measured-style packet, built the way the measured run builds them: git-tracked files only, no .git, acceptance excluded, packet id from the withheld salt.", + "why": "The panel was calibrated on packets built by copying whole trees. The measured packets are narrower and built by different code, and nobody had checked that a judge can answer on one at all.", + "why_no_unit_test_would_have_found_them": "Neither component was wrong. The isolation isolated and the packet builder built. What was wrong was the context the two of them together put the judge in, and the only way to see it is to read what a judge actually said." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/packet-size-and-reading.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/packet-size-and-reading.json new file mode 100644 index 00000000..c3285a36 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/packet-size-and-reading.json @@ -0,0 +1,28 @@ +{ + "a_difference_between_calibration_and_measured_packets": "Calibration packets were built by copying the tree; measured packets are built from `git ls-files`, so build output and untracked files are excluded. The measured packets are therefore narrower than the ones the panel was calibrated on -- median 115 files against 354 for the largest measured one, but 15.5 MB against 4.3 MB. The narrowing removes noise rather than evidence, since an agent's work lands in tracked files, but it is a difference in the instrument between selection and measurement and is recorded as one.", + "document_id": "cdeb-fresh-v8-packet-size-and-reading", + "measured": { + "calibration_packet_files": { + "max": 355, + "median": 115 + }, + "calibration_packet_tree_bytes": { + "max": 15488793, + "median": 15474970, + "min": 4336372 + }, + "measured_packet_files": { + "agent-operator-score": 354, + "gitseed": 108 + }, + "measured_packet_tree_bytes": { + "agent-operator-score": 4335053, + "gitseed": 2358808 + } + }, + "not_a_blocker": "Nothing here stops the measured run. It bounds what the judgements can be said to establish, and belongs in RESULT.md's limitations.", + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_this_means_for_the_instrument": "A packet tree is millions of tokens if read whole, and no judge reads it whole. The calibration event streams show what actually happens: a judge lists the directory, reads decision.txt, task.txt and diff.patch, and then opens the few source files it decides are relevant. The packet is a working directory, not a prompt.", + "why_that_is_a_limitation_and_not_a_defect": "It is how the calibration was run and the panel reached 43/47 unanimity that way, so the judges were selected under the same regime they will be measured under. But 'the judge saw the final tree' is a weaker statement than it sounds: the judge saw the diff and whatever else it chose to open. A violation visible only in a file no judge opened would be scored compliant." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/preflight/scoring-path-control.json b/bench/cdeb/studies/cdeb-fresh-v8/preflight/scoring-path-control.json new file mode 100644 index 00000000..a743427d --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/preflight/scoring-path-control.json @@ -0,0 +1,43 @@ +{ + "badA_note": "badA scores regression_new=4 on agent-operator-score even in the corrected order. That is a property of the control rather than the harness -- v6 built it to pass its own acceptance, not the suite -- and it is recorded because an FVR outcome may be rarer than the design assumes.", + "document_id": "cdeb-fresh-v8-scoring-path-control", + "fix": "run-episode.py measures regression on the agent's tree before installing the acceptance, then installs it and runs task acceptance.", + "method": "For one candidate per repository: materialise the snapshot, apply the v7 rebuilt goodA patch (an implementation that makes the acceptance pass), and score it exactly as an episode would. No coding agent runs.", + "not_a_pilot": "No coding agent ran and no outcome was produced.", + "question": "Can the scoring path produce a positive outcome at all, on real trees?", + "result": { + "agent-operator-score": { + "base": { + "acceptance_exit": 1, + "functional_pass": false, + "regression_new": 0 + }, + "goodA": { + "acceptance_exit": 0, + "functional_pass": true, + "regression_new": 0 + } + }, + "gitseed": { + "base": { + "acceptance_exit": 1, + "functional_pass": false, + "regression_new": 0 + }, + "goodA": { + "acceptance_exit": 0, + "functional_pass": true, + "regression_new": 0 + } + } + }, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_it_found": { + "consequence": "In the contaminated order, goodA -- an implementation built to pass -- scored regression_new=5 and functional_pass=False. Every agent-operator-score episode would have been functionally failed however well the agent did.", + "defect": "Regression acceptance was measured after the acceptance test was installed into the tree.", + "mechanism": "`node --test` with no arguments discovers every test file, so installing the acceptance adds it to the regression suite. On agent-operator-score the effect is not one extra failure: adding a file makes six census-style tests fail that pass without it.", + "why_no_unit_test_would_catch_it": "Each step is correct in isolation. The defect is in the order, and only a check that asks for a positive outcome end to end can see it." + }, + "why_it_had_to_be_asked": "Four defects found while building the runner all had the same shape -- a zero that reads as a result. Hunting them one at a time finds the ones you think of. Asking whether the path can score a pass at all covers the family." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/red-team/round-c.json b/bench/cdeb/studies/cdeb-fresh-v8/red-team/round-c.json index 433049a5..503e977d 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/red-team/round-c.json +++ b/bench/cdeb/studies/cdeb-fresh-v8/red-team/round-c.json @@ -12,7 +12,11 @@ "verified": true }, "claim-gate input provenance": { - "resolution": "partial. evaluate_gate now refuses to answer without a provenance map naming where each input came from; the simulation and unit controls pass allow_unsourced=True because their inputs are invented. This does not derive the numbers from sealed artifacts -- that builder belongs with the measured rows, which do not exist -- but a hand-assembled run can no longer look identical to a derived one.", + "half_one": "evaluate_gate refuses to answer without a provenance map naming each input's origin; the simulation and unit controls pass allow_unsourced=True because their inputs are invented", + "half_two": "gate-inputs.py derives 13 of the 25 conditions' inputs from sealed artifacts and returns the origin of each, and refuses a partial seal", + "recorded_in": "preflight/gate-input-coverage.json", + "remainder": "12 conditions need analysis output or operator records that do not exist before the measured run. An underived input fails the gate rather than defaulting, so a partial derivation cannot reach the claim -- on a synthetic full seal the gate failed on 15 conditions.", + "resolution": "addressed in two halves, with the remainder stated rather than implied", "severity_as_raised": "P1", "verified": true }, diff --git a/bench/cdeb/studies/cdeb-fresh-v8/regression-baseline.json b/bench/cdeb/studies/cdeb-fresh-v8/regression-baseline.json new file mode 100644 index 00000000..370ab85b --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/regression-baseline.json @@ -0,0 +1,39 @@ +{ + "document_id": "cdeb-fresh-v8-regression-baseline", + "repositories": { + "agent-operator-score": { + "command": "node --test", + "exit_code": 1, + "expected_failure_count": 10, + "expected_failures": [ + "ADR-0003 correction invalidates the D0-001 bounded-RED planning acceptance", + "D0 identity control-plane paths are allowed while unrelated source is rejected", + "D0-002 RED census correction invalidates the prior acceptance and renews exact prerequisites", + "banned-wording-guard-covers-commit-messages", + "banned-wording-guard-is-load-bearing", + "current registry invalidates the stale D0-001 batch and requires renewed external review", + "planning contract validator reports the truthful structural census", + "removeTempFixture refuses a path outside the OS temp root", + "removeTempFixture refuses when an INTERMEDIATE component symlinks out of the temp root", + "skeleton-source-requires-an-owning-ticket" + ], + "green_at_baseline": false, + "recorded_total": 604, + "snapshot_commit": "2faafc35bfb26d5b276be1ded4742b24607d247d" + }, + "gitseed": { + "command": "python3 -m pytest -q", + "exit_code": 0, + "expected_failure_count": 0, + "expected_failures": [], + "green_at_baseline": true, + "recorded_total": 321, + "snapshot_commit": "222378defcb5d2d519184b6f23146abac631faba" + } + }, + "schema_version": 1, + "study_id": "cdeb-fresh-v8", + "what_this_is": "The tests already failing on each repository's pristine snapshot. Regression acceptance means no failure outside this set; a new one is a regression and one of these is the snapshot.", + "why_it_is_frozen_rather_than_recomputed": "Recomputing per episode would let a flaky test join the baseline the first time it failed, which is the same as having no baseline.", + "why_it_is_needed": "The frozen population records the command and the test total and nothing about the baseline's own failures. Scored by exit code, every agent-operator-score episode fails regression whatever the agent does, which would zero that repository in an equal-weight estimand and make section 27's AOS condition unreachable." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/suppression-identity.json b/bench/cdeb/studies/cdeb-fresh-v8/suppression-identity.json new file mode 100644 index 00000000..2aa32cc2 --- /dev/null +++ b/bench/cdeb/studies/cdeb-fresh-v8/suppression-identity.json @@ -0,0 +1,130 @@ +{ + "all_resolved": true, + "candidates_without_a_record_id": [ + "v4-34aef026d81c2f6b" + ], + "counts": { + "by_record_id": 16, + "by_storage_locator": 1, + "total": 17, + "unresolved": 0 + }, + "document_id": "cdeb-fresh-v8-suppression-identity", + "identities": [ + { + "candidate_id": "v4-002ffd1e428c572a", + "kind": "record-id", + "record_id": "r-e0b001", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-0ecd7426eebc1cab", + "kind": "record-id", + "record_id": "r-gsf501", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-34aef026d81c2f6b", + "commit_sha_prefix": "f9a62917", + "decision_ordinal": 0, + "kind": "storage-locator", + "matched_field": "the record's sha/shas. A record carries no storageLocator field, so matching on one finds nothing and suppresses nothing -- and an arm that suppresses nothing is indistinguishable from ON while reporting as SUPPRESSED.", + "source": "preflight/manipulation-preflight.json structural_identity.fallback_for_one_candidate", + "storage_kind": "commit-trailer", + "storage_locator": "commit:f9a62917", + "why_it_is_unique": "the commit it names addresses exactly one record of the 66 its path scope returns, established by the section 15 preflight" + }, + { + "candidate_id": "v4-377f04276465b59d", + "kind": "record-id", + "record_id": "r-gsb108", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-77e1745655a235ce", + "kind": "record-id", + "record_id": "r-evid610", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-84cd6d391ac2fa6d", + "kind": "record-id", + "record_id": "r-f8adapter", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-8f24735524874167", + "kind": "record-id", + "record_id": "r-e0b003", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-8fc3d2ec14b1c078", + "kind": "record-id", + "record_id": "r-gs0006", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-9b42b1951da730e1", + "kind": "record-id", + "record_id": "r-e0a001", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-c61d7c943edd8cff", + "kind": "record-id", + "record_id": "r-e0b001b", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-cadfb63755c3f504", + "kind": "record-id", + "record_id": "r-gs5b02", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-ce2adee3c134ab03", + "kind": "record-id", + "record_id": "r-e0b001b", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-dd4a74ba2b628991", + "kind": "record-id", + "record_id": "r-e0a001", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-e7587b2b65750306", + "kind": "record-id", + "record_id": "r-e0a001b", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-ed878960135ff45a", + "kind": "record-id", + "record_id": "r-f8replay", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-f3c960a48273132c", + "kind": "record-id", + "record_id": "r-gsf501", + "source": "task-population.json source_decision_packet.record_id" + }, + { + "candidate_id": "v4-f901052615fa3aee", + "kind": "record-id", + "record_id": "r-f8adapter", + "source": "task-population.json source_decision_packet.record_id" + } + ], + "never_substring": "No entry matches on ruling or reason text. Removing whatever shares wording is a different manipulation from the one registered, and it would take the ruling out of records the study is not suppressing.", + "schema_version": 1, + "sources": { + "preflight/manipulation-preflight.json": "c9c5be10d9d05d02611d7f007fa7e8a53faeb949f4bcc0b1627d905e0ddbd9fa", + "task-population.json": "c17d37bad8e9a8208a8076d6987b17eda6213887dd6018b2c199838cf0ceb0df" + }, + "study_id": "cdeb-fresh-v8", + "what_this_is": "The structured identity the SUPPRESSED arm removes, for each of the seventeen. The episode runner refuses an assignment whose identity is unresolved, and refuses a suppression that removes a number of records other than one." +} diff --git a/bench/cdeb/studies/cdeb-fresh-v8/transitions.jsonl b/bench/cdeb/studies/cdeb-fresh-v8/transitions.jsonl index 25d7fd8c..f6dfb737 100644 --- a/bench/cdeb/studies/cdeb-fresh-v8/transitions.jsonl +++ b/bench/cdeb/studies/cdeb-fresh-v8/transitions.jsonl @@ -13,3 +13,16 @@ {"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "RELIABILITY_METRICS_IMPLEMENTED", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"raised_by": "red-team round B, reliability overclaim"}, "outputs": {"analysis sha256": "542e3618ad34b3e57733e9d367e0d3e83760111da3f1fca1cd2d317934081db8", "mutations": "18/18 caught"}, "checks": ["Gwet AC1, Fleiss kappa, pairwise and three-way agreement implemented per section 10", "twelve unit controls against values worked out by hand, not read off the code", "the prevalence paradox is a control: AC1 > 0.9 where kappa is lower on the same data", "three mutations cover the three metrics"], "deviations": []} {"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "OWNER_RULING_ARM_MAPPING", "actor": "OWNER", "at": "2026-08-28T00:00:00Z", "inputs": {"raised_by": "red-team round B, assignment revealed before seal"}, "outputs": {"ruling": "section 21.4 governs role access, not public availability"}, "checks": ["arm order recomputed from the committed seed and matched, so the finding's fact was confirmed before the ruling was applied"], "deviations": ["v8-d005"]} {"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "OWNER_RULINGS_RED_TEAM_P1", "actor": "OWNER", "at": "2026-08-28T00:00:00Z", "inputs": {"raised_by": "red-team rounds A, B and C"}, "outputs": {"headline": "narrowed to name the benchmark in the sentence itself", "calibration": "frozen corpus kept; confound bounds the evidence tier under section 26", "PRD sha256": "80f74ea23788fef1a6a5f0cc7696f58655bf3656cf2e97e1ef6775a236f485d7"}, "checks": ["both rulings recorded with what they permit and what they forbid", "the calibration ruling names what it bounds (selection validity) and what it does not (measured agreement, computed on episodes with no origin split)"], "deviations": ["v8-d009", "v8-d010"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "EPISODE_RUNNER_DRY_RUN", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"candidates": 17, "coding_agent_runs": 0}, "outputs": {"dry-run sha256": "c38a31a57d14b0cc92565d37ffe525e831eab200206d50890b0b3590afd3a732", "suppression-identity sha256": "5ad5de116fce85ec8e8a7bad65a901b58875c9a5df5fc5e4ef66a73762202398", "passing": "17/17"}, "checks": ["each snapshot materialised from its bundle at the pinned commit", "SUPPRESSED removes exactly the target and every other record survives", "8 of 17 candidates have record ids where one is a prefix of another, so exact identity is load-bearing for nearly half the population", "the refusal was exercised: a target not in scope removes 0 and is refused, a null target removes 41 and is refused", "no coding agent ran and no outcome was produced, so this is not a pilot"], "deviations": ["v8-d011"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "EPISODE_PACKET_BUILDER", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"coding_agent_runs": 0}, "outputs": {"episode_packet sha256": "bc96a5b285f119e8cd5f9529c1825ceb1090934b69f95f5e9c546fdf8f2976ab", "run-episode sha256": "2759c28592818cbf35f5019f910eeb9b93c1da66ef6931bb6ad7b6dd4a1539b7"}, "checks": ["the packet is built at step 15, before the tree is destroyed at step 16", "inclusion is a whitelist of git-tracked files, so anything the harness starts writing later stays out by default rather than arriving by default", "the acceptance test is excluded from the packet as well as hidden during the run", ".git is absent from the packet: branch names and commit messages name the arm", "the cue scan covers the diff and the files the agent changed, because the base tree is identical across arms; a base-tree marker count is recorded as context", "verified on two real trees: an unchanged tree scores 0 cues against 71 base markers, and an agent-written line naming its arm scores 3"], "deviations": []} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "GATE_INPUT_BUILDER", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"raised_by": "red-team round C, claim-gate input provenance"}, "outputs": {"gate-inputs sha256": "07cdf98301833b65575156a9ba1080ddfec4bb6421eb1812c4abf9c127a44a8e", "coverage sha256": "4934de0c54434cd742775a25b272c71397c65f00c9123c0f07bc80f8474123d5", "derived": "13 of 25"}, "checks": ["a partial seal is refused rather than answered", "every derived input carries the file or computation it came from", "an underived input fails the gate rather than defaulting to a pass", "exercised on a synthetic 340-row seal; no measured row exists"], "deviations": []} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "ANALYST_DRY_RUN", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"sealed_rows": 340, "judgements": 1020, "data": "synthetic"}, "outputs": {"comparison sha256": "4ae52e6bf968f359d7714e968825ba198f5f8fac1ab5db86b2c2bb6529288df6", "deterministic_matched": "11 of 12"}, "checks": ["ANALYST-B was a fresh session in a different model family with the frozen SAP and the sealed data and no reference implementation; it wrote its own analysis", "ANALYST-A's numbers were computed and written down before B's answer was read", "panel label counts, the primary effect, both repository effects, both FVRs, all four reliability metrics and both completion rates agreed", "the single deterministic mismatch was RBDR, which the specification never defined", "the Monte Carlo gap was measured rather than assumed: 2.8e-3 and 1.5e-3"], "deviations": ["v8-d012", "v8-d013"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "REGRESSION_BASELINE_FROZEN", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"repositories": 2, "coding_agent_runs": 0}, "outputs": {"regression-baseline sha256": "58459e166eaddebb70f51d0379f4d976f2bd39075c4490bfa1b0b3a8bb64a40d", "agent-operator-score": "10 expected failures of 604", "gitseed": "green at baseline"}, "checks": ["the failure set was measured three times on freshly materialised trees and was identical each time, so the baseline is not flaky", "comparison is by test name rather than by count, so a swap is not a pass", "verified in both directions: pristine passes, a broken source file fails"], "deviations": ["v8-d014"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "ACCEPTANCE_COMMANDS_FROZEN", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"candidates": 17, "coding_agent_runs": 0}, "outputs": {"acceptance-commands sha256": "3d29a0f88f97f1bb388b3f03e52c5ee5a424fc36e1be26850bbab8324cf5badf", "normalised": 4, "runnable": "17/17", "fails_on_base": "17/17"}, "checks": ["each command verified on a freshly materialised tree with the acceptance installed", "runnable means exit is not 126 or 127; the two are distinguished from a failure", "fails-on-base re-established here rather than read from the v6 record", "every normalisation recorded per candidate with its before and after"], "deviations": ["v8-d015"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "PRE_START_FAILURE_DISTINGUISHED", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"raised_by": "a systematic look for a fourth defect of the same family"}, "outputs": {"run-episode sha256": "afa582cdb0728f6d0171311161d4a28c0c7bee98446f49fe45fd7a73781cc678"}, "checks": ["the signal is the agent's event stream, not the exit code", "verified in both directions on a real stream and a stubbed 401", "the runner records retry eligibility and never retries on its own"], "deviations": ["v8-d016"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "SCORING_PATH_CONTROL", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"coding_agent_runs": 0, "candidates_checked": 2}, "outputs": {"control sha256": "9322287f3eda62dd648dacf956fff926a5fe0aecfedb5644b6702c9ebcadcef0", "run-episode sha256": "c9e600db3e66b1d968b85f044c04ae67ee039f1b7385f32b78c5b6ed0df6c712"}, "checks": ["the question was whether a pass can be scored at all, not whether a particular defect is present", "base scores false and a passing implementation scores true in both repositories", "the defect it found is in the order of the steps, which no unit test on an individual step could see"], "deviations": ["v8-d017"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "FVR_FEASIBILITY_CHECKED", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"controls": 17, "coding_agent_runs": 0}, "outputs": {"feasibility sha256": "096eea26fdd800f54619649146db947d6308b1c14d22339444533569c4b4cece", "functionally_passing_violations": "13/17"}, "checks": ["all 17 badA controls pass their task acceptance", "13 introduce no new regression and are therefore functionally passing violations", "the four that do are all agent-operator-score and are named", "this bounds possibility, not the measured rate"], "deviations": []} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "DELIVERY_CONTROL", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"candidates": 17, "coding_agent_runs": 0}, "outputs": {"delivery-control sha256": "e94bcad0f76530b26ce11b41a6cf033368e98e43b33b8834e5041ea150553784", "on_carries_target": "17/17"}, "checks": ["the ON payload renders a Ruled-out for all seventeen", "the two arms deliver different text for all seventeen", "payload size and record count recorded per repository, because the treatment's salience differs by an order of magnitude between them"], "deviations": []} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "HARNESS_SMOKE_END_TO_END", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"assignments": 2, "coding_agent_runs": 0, "model_calls": 0}, "outputs": {"harness-smoke sha256": "18b69134477ba0e81149157399c028388b69d6ca14482832ed34e48406345535", "run-episode sha256": "f51f1f93ac7d99f4d05c1c9f4dfdeed2562873d80bdb184d88d726d14ec3eac2", "checks": "26/26"}, "checks": ["the production run() was called, not a copy of it", "both arms of one pair, because the pair is the unit", "a name collision found on the way: `started` held a timestamp and was then overwritten with a boolean; it worked only because the elapsed time was computed first"], "deviations": []} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "JUDGE_PATH_PROBE", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"judgements": 2, "measured_episodes": 0}, "outputs": {"probe sha256": "7c5c5963f5a6c28c8acacb8c1ce56b057ca06c88f5b07315fe3e8c990b88c617", "judge-run sha256": "4f711c02a433eed1fd8475ca17e50d389fb79d724514554f4ed4423c58df8323", "harness smoke": "28/28"}, "checks": ["a judge answered on a measured-style packet and matched the key", "the packet id it reported was wrong, and is now right", "the harness smoke asserts the packet directory name"], "deviations": ["v8-d018"]} +{"schema_version": 1, "study_id": "cdeb-fresh-v8", "transition": "ACCIDENTAL_EPISODE_START_RECORDED", "actor": "ORCHESTRATOR", "at": "2026-08-28T00:00:00Z", "inputs": {"assignments_touched": 2, "rows_written": 0, "files_modified": 0}, "outputs": {"incident sha256": "8de55fa073b54ad121aec7ba760a6a72819d61254c84e039e198ebe16e75c146", "batch sha256": "3a710bad4e28ed815135096ee4279a13015b9bdd3d2e16dc74321855bcee952d"}, "checks": ["the rollouts were read to establish that a model was reached, rather than assumed either way", "zero files were modified in either tree; both agents were still reading", "the 105 MB of working directories were destroyed and the digests kept", "batch.py --plan verified to print and exit 0; without it, exit 2"], "deviations": ["v8-d019"]} diff --git a/test/cdeb-v8-readiness.test.ts b/test/cdeb-v8-readiness.test.ts index 2ee09af8..0d0ece50 100644 --- a/test/cdeb-v8-readiness.test.ts +++ b/test/cdeb-v8-readiness.test.ts @@ -379,12 +379,24 @@ describe("the analysis was proven before any episode existed", () => { } }); - it("lets no synthetic scenario reach the strong claim", () => { - // Including the one with exactly zero generated effect that cleared both - // headline statistical conditions by chance. A gate that a synthetic run can - // pass is not a gate. - for (const [, result] of Object.entries(scenarios)) { - expect(result.strong_claim_allowed).toBe(false); + it("blocks every scenario that was not generated with a positive effect", () => { + // This used to assert that no scenario reached the claim, which was true only + // by accident: RBDR was implemented before it was defined, and the invented + // formula happened to fall below the 50% threshold. With the registered + // pair-based definition, a scenario generated with a large positive effect and + // every non-statistical condition held at passing does reach the claim -- which + // is the gate working. What must never pass is a scenario with no effect or a + // harmful one. + const mustBeBlocked = [ + "exact_null", + "known_negative", + "completion_degraded", + "high_indeterminate", + "suppressed_fvr_zero", + ]; + for (const name of mustBeBlocked) { + expect(scenarios[name], name).toBeDefined(); + expect(scenarios[name]!.strong_claim_allowed, name).toBe(false); } });