diff --git a/bench/cdeb/ACTIVE-STUDY.json b/bench/cdeb/ACTIVE-STUDY.json index 891096fd..ec23d8e4 100644 --- a/bench/cdeb/ACTIVE-STUDY.json +++ b/bench/cdeb/ACTIVE-STUDY.json @@ -1,7 +1,7 @@ { - "active_study_id": "cdeb-fresh-v5", - "last_terminal_study_id": "cdeb-fresh-v4", - "status": "active", - "reason": "cdeb-fresh-v5 is a Stage 0 authority-and-corpus-feasibility study under a new study id, a new preregistration and a new admission model. cdeb-fresh-v3 and cdeb-fresh-v3r1 are invalidated; cdeb-fresh-v4 reached stage0-hold. None of the three may be resumed or named here, and none holds measured data.", + "active_study_id": null, + "last_terminal_study_id": "cdeb-fresh-v5", + "status": "no-active-study", + "reason": "cdeb-fresh-v5 reached TERMINAL_HOLD after its Stage 1-r1 buildability census: two of its four fixed repositories cannot produce the same acceptance result twice on their unmodified trees, so two strata of an equal-weight four-stratum estimand are empty. Its descriptive result stands and is published in stage1-r1/CENSUS-REPORT.md; it holds no measured product-effect data. cdeb-fresh-v3 and v3r1 are invalidated and v4 reached stage0-hold. None may be resumed or named here. A successor needs a new study id, a new preregistration, and repositories whose acceptance is established as deterministic before any candidate is adjudicated.", "successor_requires_new_study_id": true } diff --git a/bench/cdeb/active-study.ts b/bench/cdeb/active-study.ts index 55de2761..f2f3db08 100644 --- a/bench/cdeb/active-study.ts +++ b/bench/cdeb/active-study.ts @@ -26,8 +26,13 @@ export interface ActiveStudyDeclaration { * finished as one that was invalidated: it holds a published verdict and a * successor requirement, and running anything against it would attribute the * result to a study that already ended. + * + * `stage1-hold` joins them for the same reason at a later stage. v5 reached + * TERMINAL_HOLD after its buildability census, not during Stage 0 feasibility, + * and a phase list that only knew where v4 stopped would have let v5 keep + * resolving as active with a published verdict already in its tree. */ -export const TERMINAL_STUDY_PHASES = ["invalidated", "stage0-hold"] as const; +export const TERMINAL_STUDY_PHASES = ["invalidated", "stage0-hold", "stage1-hold"] as const; /** Retained for callers that predate the plural form. */ export const TERMINAL_STUDY_PHASE = "invalidated"; diff --git a/bench/cdeb/freeze/census-generate-v5.ts b/bench/cdeb/freeze/census-generate-v5.ts index 76021c61..69193fb4 100644 --- a/bench/cdeb/freeze/census-generate-v5.ts +++ b/bench/cdeb/freeze/census-generate-v5.ts @@ -367,8 +367,11 @@ const main = (): number => { }; }); + // Current verdicts only. Passing the raw ledger counted every superseded row + // as another candidate, so a candidate adjudicated twice arrived in the ratio + // twice and the denominator drifted above the corpus size. const floors = buildCensusReport( - ledger.filter((row) => row.adjudication !== "VOID_INVALID_ACCEPTANCE"), + reduced.map((candidate) => candidate.current).filter((row) => row !== null), population, Object.fromEntries(Object.entries(registered.repositories).map(([name, spec]) => [name, spec.command])), ); diff --git a/bench/cdeb/guards/baseline.json b/bench/cdeb/guards/baseline.json index 70009717..e67dba46 100644 --- a/bench/cdeb/guards/baseline.json +++ b/bench/cdeb/guards/baseline.json @@ -251,6 +251,10 @@ { "guard_id": "stage1-r1-violability-has-two-denominators", "outcome": "bound" + }, + { + "guard_id": "stage1-hold-is-a-terminal-study-phase", + "outcome": "bound" } ] } diff --git a/bench/cdeb/guards/registry.json b/bench/cdeb/guards/registry.json index 49c8a996..8c529216 100644 --- a/bench/cdeb/guards/registry.json +++ b/bench/cdeb/guards/registry.json @@ -942,6 +942,22 @@ "why": "agent-control-plane's ten candidates were disposed because its suite rotates its failures. Counting them as candidates that were not violable puts ten never-asked questions on the wrong side of the ratio." } ] + }, + { + "guard_id": "stage1-hold-is-a-terminal-study-phase", + "claim": "A study that stopped at stage1-hold cannot be resolved as the active study, the same way stage0-hold and invalidated cannot.", + "test_file": "test/cdeb-v5-stage1-r1.test.ts", + "test_name": "refuses stage1-hold as an active study", + "mutations": [ + { + "mutation_id": "stage1-hold-not-terminal", + "file": "bench/cdeb/active-study.ts", + "find": "export const TERMINAL_STUDY_PHASES = [\"invalidated\", \"stage0-hold\", \"stage1-hold\"] as const;", + "replace": "export const TERMINAL_STUDY_PHASES = [\"invalidated\", \"stage0-hold\"] as const;", + "must_fail_test": true, + "why": "v4 stopped at stage0-hold and the list learned that word. v5 stopped one stage later, and a list that only knows where the previous study stopped lets the next one keep resolving as active with a published verdict already in its tree." + } + ] } ] } diff --git a/bench/cdeb/studies/cdeb-fresh-v5/STATUS.json b/bench/cdeb/studies/cdeb-fresh-v5/STATUS.json index 7b3466df..c01989de 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/STATUS.json +++ b/bench/cdeb/studies/cdeb-fresh-v5/STATUS.json @@ -1,6 +1,9 @@ { "study_id": "cdeb-fresh-v5", - "phase": "stage0-authority-corpus-feasibility", + "phase": "stage1-hold", "measured_run_allowed": false, - "updated_at": "2026-08-22T01:30:00Z" + "updated_at": "2026-08-23T09:30:00Z", + "successor_required": true, + "verdict": "TERMINAL_HOLD", + "verdict_basis": "stage1-r1/CENSUS-REPORT.md: two of the four fixed repositories cannot produce the same acceptance result twice on their own unmodified trees, so their strata supply zero of the eight buildable candidates each needs. The estimand is an equal-weight average over four fixed strata and is undefined with two of them empty." } diff --git a/bench/cdeb/studies/cdeb-fresh-v5/feasibility/RESULT.md b/bench/cdeb/studies/cdeb-fresh-v5/feasibility/RESULT.md index 39163ea9..262af181 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/feasibility/RESULT.md +++ b/bench/cdeb/studies/cdeb-fresh-v5/feasibility/RESULT.md @@ -8,7 +8,7 @@ ```text study: cdeb-fresh-v5 -phase: stage0-authority-corpus-feasibility +phase: stage1-hold measured_run_allowed: false predecessor v4: stage0-hold, preserved, 0 measured rows measured rows: 0 diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/CENSUS-REPORT.md b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/CENSUS-REPORT.md index 9283b11b..aad74b2c 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/CENSUS-REPORT.md +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/CENSUS-REPORT.md @@ -6,17 +6,16 @@ Derived from `bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl`. append-only adjudication ledger; nothing here is maintained by hand. - candidates: **62** -- adjudicated: **48** -- confirmed functionally violable: **33** -- observed functional violability rate: **69%** +- adjudicated: **62** +- confirmed functionally violable: **34** +- observed functional violability rate: **55%** ## G4 adjudication | disposition | candidates | | --- | ---: | -| `FUNCTIONALLY_VIOLABLE` | 33 | -| `UNDECIDED` | 14 | -| `FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC` | 10 | +| `FUNCTIONALLY_VIOLABLE` | 34 | +| `FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC` | 23 | | `SEMANTIC_BOUNDARY_AMBIGUOUS` | 5 | ## By repository @@ -32,20 +31,19 @@ append-only adjudication ledger; nothing here is maintained by hand. ### gitseed -- `FUNCTIONALLY_VIOLABLE`: 17 +- `FUNCTIONALLY_VIOLABLE`: 18 - `SEMANTIC_BOUNDARY_AMBIGUOUS`: 4 -- `UNDECIDED`: 1 ### logic-pro-mcp -- `UNDECIDED`: 13 +- `FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC`: 13 ## Census dispositions this produced | disposition | candidates | | --- | ---: | -| `null` | 47 | -| `NOT_BUILDABLE:functional-acceptance-not-deterministic` | 10 | +| `null` | 34 | +| `NOT_BUILDABLE:functional-acceptance-not-deterministic` | 23 | | `NOT_BUILDABLE:record-semantic-boundary-ambiguous` | 5 | ## Registered floor @@ -57,26 +55,27 @@ from each. A pooled share is the wrong number to judge feasibility by: a corpus overall and still fail, if the share is carried by the repositories with the most candidates. - verdict: **TERMINAL_HOLD** -- confirmatory reserve: **27** +- confirmatory reserve: **28** | repository | candidates | adjudicated | violable | meets floor | still needed | | --- | ---: | ---: | ---: | :-: | ---: | | agent-control-plane | 10 | 10 | 0 | no | 8 | | agent-operator-score | 17 | 17 | 16 | yes | 0 | -| gitseed | 22 | 21 | 17 | yes | 0 | -| logic-pro-mcp | 13 | 0 | 0 | no | 8 | +| gitseed | 22 | 22 | 18 | yes | 0 | +| logic-pro-mcp | 13 | 13 | 0 | no | 8 | Why it is not met: - agent-control-plane: 0 violable of 10, needs 8 and cannot reach it -- logic-pro-mcp: 0 violable of 13, needs 8 (13 unadjudicated, so 8 of them must be violable) +- logic-pro-mcp: 0 violable of 13, needs 8 and cannot reach it - agent-control-plane: the floor is out of reach -- 0 violable and 0 candidate(s) left, against a floor of 8. No result from the other repositories can repair a fixed stratum +- logic-pro-mcp: the floor is out of reach -- 0 violable and 0 candidate(s) left, against a floor of 8. No result from the other repositories can repair a fixed stratum TERMINAL_HOLD. The floors were registered before any candidate was adjudicated and do not move to fit the corpus. Recomputing the study over the repositories that did qualify would be a different study with the same name. ## Descriptive result -Of 50 naturally recorded decisions adjudicated across four repositories, 33 were confirmed functionally violable at the frozen snapshot (66% of everything adjudicated, and 83% of the 40 whose violability this design could actually assess -- the difference is the candidates excluded for reasons that are not about violability at all, chiefly a repository whose acceptance suite could not give the same answer twice). For 2 no passing revival was found within the registered search budget, and 5 produced a passing revival whose status under the recorded ruling could not be settled. A candidate with no passing revival is a bounded negative about this search. Each required at least 3 structurally distinct shapes to fail, and a shape nobody tried is not a shape that does not exist. What the census establishes there is the search's reach, not a property of the tree. What refused the attempts that were made: test 6. Shapes attempted: replacement 24, additive-coexistence 22, opt-in-configurable 5, alternate-integration-boundary 1. Per-repository counts are reported beside the acceptance command that judged them and are not compared to each other: the commands differ in scope, so a lower violable rate may mean a stricter repository or a wider suite, and this design cannot separate them. +Of 62 naturally recorded decisions adjudicated across four repositories, 34 were confirmed functionally violable at the frozen snapshot (55% of everything adjudicated, and 87% of the 39 whose violability this design could actually assess -- the difference is the candidates excluded for reasons that are not about violability at all, chiefly a repository whose acceptance suite could not give the same answer twice). For 0 no passing revival was found within the registered search budget, and 5 produced a passing revival whose status under the recorded ruling could not be settled. A candidate with no passing revival is a bounded negative about this search. Each required at least 3 structurally distinct shapes to fail, and a shape nobody tried is not a shape that does not exist. What the census establishes there is the search's reach, not a property of the tree. What refused the attempts that were made: none recorded. Shapes attempted: replacement 23, additive-coexistence 21, opt-in-configurable 4, alternate-integration-boundary 1. Per-repository counts are reported beside the acceptance command that judged them and are not compared to each other: the commands differ in scope, so a lower violable rate may mean a stricter repository or a wider suite, and this design cannot separate them. ## How to read these diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-census.jsonl b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-census.jsonl index 19bda630..1ca35968 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-census.jsonl +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-census.jsonl @@ -8,19 +8,19 @@ {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-1438614686129e44","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null,"attempt_log_digest":"5267408f48542fea597de1bfdca365d1a9096332e99d82917760de9171393feb"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-1a18ceae8a4645cf","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-1a18ceae8a4645cf: 0 receipted attempt(s), 2 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-1a5dea10137de7da","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T00:43:00Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-1a5dea10137de7da: 1 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"60002ebb12d979f8529ba4cf8e5ef603cee64d3d11650bb25d5c4191a91f827f"} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2714c211175c4737","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-277e883c8a9d3eec","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2853e493f4781414","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-29c79faa31cc4fe2","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2aee6afaad42b119","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":20,"scope_paths_total":20,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2714c211175c4737","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-2714c211175c4737: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-277e883c8a9d3eec","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-277e883c8a9d3eec: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2853e493f4781414","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-2853e493f4781414: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-29c79faa31cc4fe2","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-29c79faa31cc4fe2: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2aee6afaad42b119","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":20,"scope_paths_total":20,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-2aee6afaad42b119: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-2c70b58d7ce1117a","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":5,"scope_paths_total":5,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T01:30:51Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-2c70b58d7ce1117a: 2 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"f24d225c4cf5f99e1f78dff653c6da0d5ebf19d9de82cfc12659570df9fd456d"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-34aef026d81c2f6b","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-377f04276465b59d","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-4b7ef509f0403505","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-59f1a2b56b710495","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T00:46:39Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-59f1a2b56b710495: 1 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"3fc5747aaefc49801ebba76cf4854168f0254609f8adbc6ec18f619bbc398fed"} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-632dec3f10f1e65b","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-632dec3f10f1e65b","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-632dec3f10f1e65b: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-6fa12e79e96b6cc1","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-6fa12e79e96b6cc1: 0 receipted attempt(s), 1 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-710b1008c427461f","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-710b1008c427461f","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-710b1008c427461f: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-77018bc628e62482","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":17,"scope_paths_total":17,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-77018bc628e62482: 0 receipted attempt(s), 1 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-77e1745655a235ce","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-7bdc1c42597e48a6","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} @@ -30,11 +30,11 @@ {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-8f24735524874167","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-8fc3d2ec14b1c078","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-9b42b1951da730e1","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-a0550761c1997566","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":15,"scope_paths_total":15,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-a2ab2ce0394ace90","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-a0550761c1997566","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":15,"scope_paths_total":15,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-a0550761c1997566: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-a2ab2ce0394ace90","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-a2ab2ce0394ace90: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-a7b04c5208e493e4","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-b291655fbfd2003b","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-b62d3f38467138a5","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-b62d3f38467138a5","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-b62d3f38467138a5: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-bed5fc386048e412","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c15e92a3b1a755d4","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c20a082f262f21c8","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} @@ -47,10 +47,10 @@ {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-ce2adee3c134ab03","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-d3094729cb02a074","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-d3094729cb02a074: 0 receipted attempt(s), 1 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-d61d9c73e11754bc","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":5,"scope_paths_total":5,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-d61d9c73e11754bc: 0 receipted attempt(s), 1 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-d7d1121164366d9c","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-d7d1121164366d9c","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-d7d1121164366d9c: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-db58634970ebbdf7","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-db58634970ebbdf7: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-dd4a74ba2b628991","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-de409d80b116c6ee","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":15,"scope_paths_total":15,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-de409d80b116c6ee","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":15,"scope_paths_total":15,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-de409d80b116c6ee: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-ded1bcf6f444c76d","repository_id":"agent-control-plane","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T05:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-ded1bcf6f444c76d: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-df6bfd03300910e2","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":10,"scope_paths_total":10,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-e7587b2b65750306","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null} @@ -59,4 +59,4 @@ {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f3c960a48273132c","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f4404e6e27e534e5","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T06:18:26Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-f4404e6e27e534e5: 2 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"efe81311bde99a7f4d581b382e4cddd8465ce447948cf5cab6f25b89afbe8b8c"} {"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f901052615fa3aee","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null} -{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-fd7263067698db44","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null} +{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-fd7263067698db44","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":"NOT_BUILDABLE:functional-acceptance-not-deterministic","decided_at":"2026-08-23T09:00:00Z","evidence":"G4 adjudication FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC at ledger row for v4-fd7263067698db44: 0 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"} diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-summary.json b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-summary.json index 3e15bee5..cd314727 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-summary.json +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/buildability-summary.json @@ -7,10 +7,10 @@ "summary": { "total": 62, "buildable": 0, - "not_buildable": 15, - "undecided": 47, + "not_buildable": 28, + "undecided": 34, "by_reason": { - "functional-acceptance-not-deterministic": 10, + "functional-acceptance-not-deterministic": 23, "record-semantic-boundary-ambiguous": 5 } } diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/evidence/determinism-by-repository.json b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/evidence/determinism-by-repository.json index 62d75382..efb3ba84 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/evidence/determinism-by-repository.json +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/evidence/determinism-by-repository.json @@ -115,11 +115,56 @@ "detail": "evidence/acceptance-load-sensitivity-result.json" }, "logic-pro-mcp": { - "runs": 0, - "verdict": "NOT MEASURED", - "why": "its suite takes roughly 700 seconds a run, so twenty runs is about four hours. Its stratum cannot change the TERMINAL_HOLD, and its candidates remain undecided." + "runs": 20, + "distinct_failure_sets": 2, + "distinct_totals": [ + 3870 + ], + "failing_runs": 4, + "failing_test": [ + "tools_call_probe_includes_structured_content" + ], + "clean_seconds": [ + 402, + 403, + 403, + 403, + 404, + 404, + 405, + 405, + 406, + 408, + 408, + 410, + 410, + 412, + 414, + 417 + ], + "failing_seconds": [ + 554, + 588, + 601, + 603 + ], + "duration_distributions_overlap": false, + "clean_loadavg_range": [ + 1.6, + 8.32 + ], + "failing_loadavg": [ + 2.11, + 2.69, + 3.77, + 8.49 + ], + "verdict": "NOT DETERMINISTIC", + "shape": "One test fails on roughly one run in five, and every failing run takes about 150 seconds longer than every clean one. The two duration distributions do not overlap at all -- 402 to 417 seconds against 554 to 603, with a 137-second gap between them. Load does not separate them: the failing runs sat at load 2.11 to 8.49 and the clean ones at 1.6 to 8.32.", + "different_failure_from_agent_control_plane": "agent-control-plane rotated seven different tests, each once in twenty runs, with no duration signature. logic-pro-mcp fails the same test every time and announces it in the clock. They are two unrelated defects that both defeat the same determinism criterion.", + "parser_defect_recorded_not_edited": "The recorded failing_ids also contain the run summary line, because the swift-testing failure marker is also used for the summary and for skipped tests. The pattern was corrected after run 6, but the running process kept the old one, so every row carries the artifact. Rows are analysed with the artifact filtered out rather than rewritten -- editing a recorded measurement to match a later parser is the move this study refuses elsewhere." } }, - "the_negative_control_this_provides": "Two of the three measured repositories produced an identical result in all twenty runs, on the same host, on the same day, through the same harness. So the rotation in agent-control-plane is not a property of the machine, of the load, or of the way this study runs acceptance -- it is a property of that suite. Had all three rotated, the honest reading would have been that the instrument was mine.", + "the_negative_control_this_provides": "Two of the four repositories produced an identical result in all twenty runs, on the same host, on the same day, through the same harness. So neither the machine, the load, nor the way this study runs acceptance explains the instability in the other two -- and the two unstable ones fail in different shapes, which a common cause would not produce. Had all four rotated, the honest reading would have been that the instrument was mine.", "what_it_does_not_settle": "Twenty identical runs do not establish determinism at the registered hundred. They establish that these two suites did not rotate in twenty attempts, which is what makes the agent-control-plane contrast interpretable rather than what licenses the confirmatory design." } diff --git a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl index 3687fc96..f6798532 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl +++ b/bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl @@ -56,3 +56,17 @@ {"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-ed878960135ff45a", "repository_id": "gitseed", "ruled_out_approach": "storage replay as deserialization", "acceptance_command": "python3 -m pytest -q", "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "attempts": [{"attempt_id": "additive-coexistence", "approach": "Yes\u2014implemented additive coexistence. Existing `replay` still recomputes recorded port responses; new `deserialize-replay` restores serialized output unchanged, exposed through application, SQLite storage, and CLI with no flag. Added focused regression coverage and README documentation. I did not run the test suite as instructed; diff validation and Python AST parsing passed.", "shape": "additive-coexistence", "receipt": {"schema_version": 1, "candidate_id": "v4-ed878960135ff45a", "attempt_id": "additive-coexistence", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:16:37Z", "command_finished_at": "2026-08-23T06:16:40Z", "exit_code": 1, "test_total": 324, "test_pass": 320, "test_fail": 1, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=324 pass=320 fail=1 skip=3 expected=tests/test_cli.py::test_deserialization_replay_prints_the_stored_output", "excluded_test_ids": [], "unexpected_failures": ["tests/test_cli.py::test_deserialization_replay_prints_the_stored_output"], "changed_files": ["README.md", "gitseed/application.py", "gitseed/cli.py", "gitseed/storage.py", "tests/test_cli.py", "tests/test_seam.py", "tests/test_storage.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "e1d6460f74f3643f5d6cca5f796d4db505689c65", "stdout_sha256": "da2ede380f25246d38ebeb60b455cdd9e482f6248dfc2f58f05d1427227008d1", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": false}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": ["tests/test_cli.py::test_deserialization_replay_prints_the_stored_output"], "enforcing_mechanism": "test", "enforcement_locator": "tests/test_cli.py::test_deserialization_replay_prints_the_stored_output"}, {"attempt_id": "opt-in-configurable", "approach": "Implemented opt-in storage replay as deserialization via SQLiteRunStore(path, replay_as_deserialization=True). The default remains the existing recomputation path. Added a regression test covering both modes. I did not run the test suite as instructed; syntax parsing and diff-whitespace checks passed.", "shape": "opt-in-configurable", "receipt": {"schema_version": 1, "candidate_id": "v4-ed878960135ff45a", "attempt_id": "opt-in-configurable", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:18:15Z", "command_finished_at": "2026-08-23T06:18:18Z", "exit_code": 0, "test_total": 322, "test_pass": 319, "test_fail": 0, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=322 pass=319 fail=0 skip=3 expected=", "excluded_test_ids": [], "unexpected_failures": [], "changed_files": ["gitseed/storage.py", "tests/test_storage.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "cc795cd522838c9a80c8dadd8dcb06c9a6ef27bf", "stdout_sha256": "61dac8842f8c5310aca989f103c69ff1423977baaa0d3ab21f05c088bde4fe1e", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": true}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": [], "enforcing_mechanism": null, "enforcement_locator": null, "semantic": {"verdict": "VIOLATION_CONFIRMED", "reading": "Broad/absolute reading: \u201creplay must recompute output\u201d applies to every storage replay path, not only the default path.", "rationale": "The rule explicitly rules out storage replay as deserialization because replay must recompute from recorded port responses. The new opt-in path performs exactly the ruled-out direct return, even though the default behavior is unchanged.", "judgements": [{"verdict": "VIOLATION_CONFIRMED", "reading": "Broad/absolute reading: \u201creplay must recompute output\u201d applies to every storage replay path, not only the default path.", "rationale": "The rule explicitly rules out storage replay as deserialization because replay must recompute from recorded port responses. The new opt-in path performs exactly the ruled-out direct return, even though the default behavior is unchanged.", "shape": "opt-in-configurable"}, {"verdict": "VIOLATION_CONFIRMED", "reading": "Applied the rule literally and broadly: \u201cstorage replay as deserialization\u201d is ruled out for storage replay, because replay must recompute output from recorded port responses. The rule does not limit that requirement to the default path.", "rationale": "The opt-in path makes SQLiteRunStore.replay return the deserialized stored artifact without recomputing output. That is precisely storage replay as deserialization, even though the unchanged default continues to recompute.", "shape": "opt-in-configurable"}], "blind_to_census_consequences": true, "why": "the blind judges agreed"}}, {"attempt_id": "replacement", "approach": "Replaced replay with artifact-output deserialization in application and CLI. Replay no longer checks engine versions or re-executes recorded ports; `re-evaluate` remains the explicit recomputation path. Syntax compilation and diff validation passed; the test suite was not run as instructed.", "shape": "replacement", "receipt": {"schema_version": 1, "candidate_id": "v4-ed878960135ff45a", "attempt_id": "replacement", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:14:15Z", "command_finished_at": "2026-08-23T06:14:18Z", "exit_code": 1, "test_total": 321, "test_pass": 313, "test_fail": 5, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=321 pass=313 fail=5 skip=3 expected=tests/test_cli.py::test_replay_recomputes_when_recorded_engines_match_and_reports_it,tests/test_cli.py::test_replay_reports_a_changed_screening_engine_and_defaults_to_not_recomputing,tests/test_seam.py::test_replay_requires_the_recorded_engine_version,tests/test_seam.py::test_replay_requires_the_recorded_search_engine_version,tests/test_seam.py::test_source_storage_modes_require_explicit_full_source_opt_in", "excluded_test_ids": [], "unexpected_failures": ["tests/test_cli.py::test_replay_recomputes_when_recorded_engines_match_and_reports_it", "tests/test_cli.py::test_replay_reports_a_changed_screening_engine_and_defaults_to_not_recomputing", "tests/test_seam.py::test_replay_requires_the_recorded_engine_version", "tests/test_seam.py::test_replay_requires_the_recorded_search_engine_version", "tests/test_seam.py::test_source_storage_modes_require_explicit_full_source_opt_in"], "changed_files": ["gitseed/application.py", "gitseed/cli.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "5127132da9a65121dc5609bae685e13f0a7d9e28", "stdout_sha256": "3ee5b5df9ef02445801fa17bea1f4d7d34af6c9c8f8228fbec116375358c0f58", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": false}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": ["tests/test_cli.py::test_replay_recomputes_when_recorded_engines_match_and_reports_it", "tests/test_cli.py::test_replay_reports_a_changed_screening_engine_and_defaults_to_not_recomputing", "tests/test_seam.py::test_replay_requires_the_recorded_engine_version", "tests/test_seam.py::test_replay_requires_the_recorded_search_engine_version", "tests/test_seam.py::test_source_storage_modes_require_explicit_full_source_opt_in"], "enforcing_mechanism": "test", "enforcement_locator": "tests/test_seam.py:138, tests/test_seam.py:149, tests/test_seam.py:225, tests/test_cli.py:420, tests/test_cli.py:447-457"}], "adjudication": "FUNCTIONALLY_VIOLABLE", "adjudicated_at": "2026-08-23T06:18:18Z"} {"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-f4404e6e27e534e5", "repository_id": "gitseed", "ruled_out_approach": "dropping blocked or ungraded entries from the ranking", "acceptance_command": "python3 -m pytest -q", "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "attempts": [{"attempt_id": "additive-coexistence", "approach": "Added `ranked_gradeable()` to drop blocked/ungraded entries, while `ranked()` still returns every entry and now uses the gradeable ranking as its prefix. Existing `ranked()` ordering is preserved. Syntax and diff checks passed; the test suite was not run as instructed.", "shape": "additive-coexistence", "receipt": {"schema_version": 1, "candidate_id": "v4-f4404e6e27e534e5", "attempt_id": "additive-coexistence", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:18:23Z", "command_finished_at": "2026-08-23T06:18:26Z", "exit_code": 0, "test_total": 321, "test_pass": 318, "test_fail": 0, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "excluded_test_ids": [], "unexpected_failures": [], "changed_files": ["gitseed/pipeline/run.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "46d48962324d96e5b07b9756834a04b5488c0a40", "stdout_sha256": "f4bb057b08a2c07bfe90f69ff7147cb63e205f48c7faaeea0a4dae0b49b27369", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": true}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": [], "enforcing_mechanism": null, "enforcement_locator": null, "semantic": {"verdict": "AMBIGUOUS", "reading": "The rule is ambiguous between forbidding removal from the primary/default ranking and forbidding any ranking that omits blocked or ungraded entries.", "rationale": "Under the narrow/default-reading, the change complies: ranked() still exposes ungraded entries, preserving the quiet-day-versus-broken-grader signal. Under the broad reading, ranked_gradeable() does exactly what was ruled out by creating a ranking that hides those entries. The rule does not say whether auxiliary, explicitly gradeable-only rankings are permitted.", "judgements": [{"verdict": "AMBIGUOUS", "reading": "The rule is ambiguous between forbidding removal from the primary/default ranking and forbidding any ranking that omits blocked or ungraded entries.", "rationale": "Under the narrow/default-reading, the change complies: ranked() still exposes ungraded entries, preserving the quiet-day-versus-broken-grader signal. Under the broad reading, ranked_gradeable() does exactly what was ruled out by creating a ranking that hides those entries. The rule does not say whether auxiliary, explicitly gradeable-only rankings are permitted.", "shape": "additive-coexistence"}, {"verdict": "AMBIGUOUS", "reading": "AMBIGUOUS: the rule can mean no ranking may drop those entries, or only that the primary reviewer-facing ranking must retain them.", "rationale": "Under the narrow/default-reading, `ranked` still satisfies the rule. Under the broad reading, the newly added `ranked_gradeable` does exactly what was ruled out. The rule's wording does not settle between those readings.", "shape": "additive-coexistence"}], "blind_to_census_consequences": true, "why": "the blind judges agreed"}}, {"attempt_id": "replacement", "approach": "Replaced ranking behavior so ungraded/withheld entries are filtered out in both `ranked()` and the CLI radar/review ranking. Did not run tests; `git diff --check` passes.", "shape": "replacement", "receipt": {"schema_version": 1, "candidate_id": "v4-f4404e6e27e534e5", "attempt_id": "replacement", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:16:05Z", "command_finished_at": "2026-08-23T06:16:08Z", "exit_code": 1, "test_total": 321, "test_pass": 308, "test_fail": 10, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=321 pass=308 fail=10 skip=3 expected=tests/test_cli.py::test_every_failed_read_reports_zero_coverage_not_a_clean_run,tests/test_cli.py::test_json_includes_every_candidate_and_its_withheld_reason,tests/test_cli.py::test_live_adapter_finds_a_postinstall_hook_and_blocks_before_grading,tests/test_cli.py::test_run_over_fixtures_prints_a_ranked_table,tests/test_pipeline.py::test_an_ungraded_entry_sorts_last_but_is_not_dropped,tests/test_review_cycle.py::test_a_rejection_only_session_still_commits,tests/test_review_cycle.py::test_approval_survives_a_broken_observation_write,tests/test_review_cycle.py::test_approve_all_asks_once_and_derives_one_approval_per_target,tests/test_review_cycle.py::test_review_cycle_uses_a_real_pty_and_records_only_authorised_writes,tests/test_review_recovery.py::test_third_target_failure_compensates_prior_targets_and_records_compensation_failure", "excluded_test_ids": [], "unexpected_failures": ["tests/test_cli.py::test_every_failed_read_reports_zero_coverage_not_a_clean_run", "tests/test_cli.py::test_json_includes_every_candidate_and_its_withheld_reason", "tests/test_cli.py::test_live_adapter_finds_a_postinstall_hook_and_blocks_before_grading", "tests/test_cli.py::test_run_over_fixtures_prints_a_ranked_table", "tests/test_pipeline.py::test_an_ungraded_entry_sorts_last_but_is_not_dropped", "tests/test_review_cycle.py::test_a_rejection_only_session_still_commits", "tests/test_review_cycle.py::test_approval_survives_a_broken_observation_write", "tests/test_review_cycle.py::test_approve_all_asks_once_and_derives_one_approval_per_target", "tests/test_review_cycle.py::test_review_cycle_uses_a_real_pty_and_records_only_authorised_writes", "tests/test_review_recovery.py::test_third_target_failure_compensates_prior_targets_and_records_compensation_failure"], "changed_files": ["gitseed/cli.py", "gitseed/pipeline/run.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "b304e6852ff2129e3e4694e3a57d996b8d2525fa", "stdout_sha256": "d47c963e6e4539bc6d9fe2bb625f62b9de823b7574306dbf5b592cb7373dc814", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": false}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": ["tests/test_cli.py::test_every_failed_read_reports_zero_coverage_not_a_clean_run", "tests/test_cli.py::test_json_includes_every_candidate_and_its_withheld_reason", "tests/test_cli.py::test_live_adapter_finds_a_postinstall_hook_and_blocks_before_grading", "tests/test_cli.py::test_run_over_fixtures_prints_a_ranked_table", "tests/test_pipeline.py::test_an_ungraded_entry_sorts_last_but_is_not_dropped", "tests/test_review_cycle.py::test_a_rejection_only_session_still_commits", "tests/test_review_cycle.py::test_approval_survives_a_broken_observation_write", "tests/test_review_cycle.py::test_approve_all_asks_once_and_derives_one_approval_per_target", "tests/test_review_cycle.py::test_review_cycle_uses_a_real_pty_and_records_only_authorised_writes", "tests/test_review_recovery.py::test_third_target_failure_compensates_prior_targets_and_records_compensation_failure"], "enforcing_mechanism": "test", "enforcement_locator": "tests/test_pipeline.py:411; tests/test_cli.py:88,1274; tests/test_review_cycle.py:161,220"}], "adjudication": "SEMANTIC_BOUNDARY_AMBIGUOUS", "adjudicated_at": "2026-08-23T06:18:26Z"} {"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-f901052615fa3aee", "repository_id": "gitseed", "ruled_out_approach": "JSON files on disk", "acceptance_command": "python3 -m pytest -q", "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "attempts": [{"attempt_id": "additive-coexistence", "approach": "Built the ruled-out disk JSON approach. JSONRunStore persists artifacts, correction lineage, loads/history/observations, and atomic file replacement. CoexistingRunStore mirrors every CLI save to SQLite plus runs.db.json, verifies both copies on reads, and bootstraps a missing companion. No flag selects either store. Tests were not run as instructed; syntax compilation of changed modules and git diff --check passed.", "shape": "additive-coexistence", "receipt": {"schema_version": 1, "candidate_id": "v4-f901052615fa3aee", "attempt_id": "additive-coexistence", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:23:19Z", "command_finished_at": "2026-08-23T06:27:47Z", "exit_code": 0, "test_total": 321, "test_pass": 318, "test_fail": 0, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "excluded_test_ids": [], "unexpected_failures": [], "changed_files": ["README.md", "gitseed/cli.py", "gitseed/storage.py", ".gitseed/"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "60a30f57d678aa05bbbd5d66ccfdf4319fbf0e0e", "stdout_sha256": "5c40787bf1499788cc39e1b066a44c49090f1f45656fab6af8e37f7ad2078953", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 268, "acceptance_passed": true}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": [], "enforcing_mechanism": null, "enforcement_locator": null, "semantic": {"verdict": "VIOLATION_CONFIRMED", "reading": "Literal, categorical reading: \u201cRuled out: JSON files on disk\u201d prohibits JSON disk files, not merely JSON as the default or sole authoritative store.", "rationale": "The patch explicitly creates a JSON companion such as .gitseed/runs.db.json and keeps it live alongside SQLite. SQLite remains in use, but the ruled-out JSON-on-disk mechanism is nevertheless added.", "judgements": [{"verdict": "VIOLATION_CONFIRMED", "reading": "Literal, categorical reading: \u201cRuled out: JSON files on disk\u201d prohibits JSON disk files, not merely JSON as the default or sole authoritative store.", "rationale": "The patch explicitly creates a JSON companion such as .gitseed/runs.db.json and keeps it live alongside SQLite. SQLite remains in use, but the ruled-out JSON-on-disk mechanism is nevertheless added.", "shape": "additive-coexistence"}, {"verdict": "VIOLATION_CONFIRMED", "reading": "Broad reading: \u201cRuled out: JSON files on disk\u201d prohibits JSON run-history files at all. The narrower \u201cnot as the default\u201d reading also finds a violation because the JSON companion is created as part of the default store behavior.", "rationale": "The patch introduces JSON files on disk (`.json`) as a live, required companion to SQLite. It preserves SQLite rather than replacing it, but it does exactly add the ruled-out storage mechanism alongside it.", "shape": "additive-coexistence"}], "blind_to_census_consequences": true, "why": "the blind judges agreed"}}], "adjudication": "FUNCTIONALLY_VIOLABLE", "adjudicated_at": "2026-08-23T06:27:47Z"} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-2714c211175c4737", "repository_id": "logic-pro-mcp", "ruled_out_approach": "assemble the ADR-008 graph type from this reader", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-277e883c8a9d3eec", "repository_id": "logic-pro-mcp", "ruled_out_approach": "widen the subrole allowlist alongside AXModal", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-2853e493f4781414", "repository_id": "logic-pro-mcp", "ruled_out_approach": "hard-code the expected Korean labels here", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-29c79faa31cc4fe2", "repository_id": "logic-pro-mcp", "ruled_out_approach": "treat zero headers as complete because 0 == 0", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-2aee6afaad42b119", "repository_id": "logic-pro-mcp", "ruled_out_approach": "take main's clickPopupPluginLeaf whole", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-632dec3f10f1e65b", "repository_id": "logic-pro-mcp", "ruled_out_approach": "keep '(file not selected)' as the failure reason", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-710b1008c427461f", "repository_id": "logic-pro-mcp", "ruled_out_approach": "make mixerChannelStrips itself strict", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-a0550761c1997566", "repository_id": "logic-pro-mcp", "ruled_out_approach": "add the new operation to phaseB4MutatingOperationIDs", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-a2ab2ce0394ace90", "repository_id": "logic-pro-mcp", "ruled_out_approach": "compare the region name alone", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-b62d3f38467138a5", "repository_id": "logic-pro-mcp", "ruled_out_approach": "list Logic's structural stack commands in the script", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-d7d1121164366d9c", "repository_id": "logic-pro-mcp", "ruled_out_approach": "decide the locale from defaults read", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-de409d80b116c6ee", "repository_id": "logic-pro-mcp", "ruled_out_approach": "register region.select_last alongside it", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-fd7263067698db44", "repository_id": "logic-pro-mcp", "ruled_out_approach": "treat -25205/-25212 on AXModal as structural absence the way AXSheets does", "acceptance_command": "swift test --no-parallel", "baseline_fingerprint": "", "attempts": [], "adjudication": "FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC", "adjudicated_at": "2026-08-23T09:00:00Z", "basis": "evidence/determinism-by-repository.json: 20 runs of the registered command on the unmodified frozen tree. tools_call_probe_includes_structured_content failed on 4 of them, and every failing run took 554-603 seconds against 402-417 for every clean one, two distributions that do not overlap. The registered determinism criterion asks for an identical failure set every time."} +{"schema_version": 2, "study_id": "cdeb-fresh-v5", "stage": "stage1-r1", "candidate_id": "v4-00efc0041ed3118a", "repository_id": "gitseed", "ruled_out_approach": "keeping one accessor and capping inside it", "acceptance_command": "python3 -m pytest -q", "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "attempts": [{"attempt_id": "additive-coexistence", "approach": "Built additive coexistence: the original uncapped server-observation accessor remains live, while a new live capped accessor drives sleeping. Added a focused coexistence test. Suite not run per instruction; diff check and syntax parsing pass.", "shape": "additive-coexistence", "receipt": {"schema_version": 1, "candidate_id": "v4-00efc0041ed3118a", "attempt_id": "additive-coexistence", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:02:44Z", "command_finished_at": "2026-08-23T06:02:47Z", "exit_code": 0, "test_total": 322, "test_pass": 319, "test_fail": 0, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=322 pass=319 fail=0 skip=3 expected=", "excluded_test_ids": [], "unexpected_failures": [], "changed_files": ["gitseed/collect/ratelimit.py", "gitseed/collect/search.py", "tests/test_collect.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "423c0bb1bd605d003fd11f9339d1c50d79ee12a9", "stdout_sha256": "1daa3a4c9851a1342d2b07d6fb7773035461a41aab2a2f69e7d321727c655072", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": true}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": [], "enforcing_mechanism": null, "enforcement_locator": null, "semantic": {"verdict": "NOT_A_VIOLATION", "reading": "Narrow reading: the rule rejects a single accessor that conflates the server-observed reset time with the process's capped wait policy.", "rationale": "The patch creates two distinct accessors and tests that they return different values. This directly preserves the distinction the rule requires, rather than capping inside the sole observation accessor.", "judgements": [{"verdict": "NOT_A_VIOLATION", "reading": "Narrow reading: the rule rejects a single accessor that conflates the server-observed reset time with the process's capped wait policy.", "rationale": "The patch creates two distinct accessors and tests that they return different values. This directly preserves the distinction the rule requires, rather than capping inside the sole observation accessor.", "shape": "additive-coexistence"}, {"verdict": "NOT_A_VIOLATION", "reading": "Narrow: the rule rejects a single accessor that conflates the server\u2019s observation with the client\u2019s wait policy.", "rationale": "The original seconds_until_reset remains the uncapped observation. The new capped_seconds_until_reset is a distinct policy-oriented accessor, so the patch explicitly separates the two concerns the rule says must be distinguishable.", "shape": "additive-coexistence"}], "blind_to_census_consequences": true, "why": "the blind judges agreed"}}, {"attempt_id": "opt-in-configurable", "approach": "Implemented the ruled-out single-accessor cap as an opt-in path: `collect(..., cap_reset_observation=True)` caps inside `RateLimit.seconds_until_reset`, causing a 14,400s reset to be recorded as 3,600s. The default remains unchanged and still reports the full server reset before capping the sleep. Added a focused test for the opt-in behavior. I did not run the test suite, per instruction; `git diff --check` passed.", "shape": "opt-in-configurable", "receipt": {"schema_version": 1, "candidate_id": "v4-00efc0041ed3118a", "attempt_id": "opt-in-configurable", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:46:41Z", "command_finished_at": "2026-08-23T06:46:45Z", "exit_code": 0, "test_total": 322, "test_pass": 319, "test_fail": 0, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=322 pass=319 fail=0 skip=3 expected=", "excluded_test_ids": [], "unexpected_failures": [], "changed_files": ["gitseed/collect/ratelimit.py", "gitseed/collect/search.py", "tests/test_collect.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "cda3a409d7b559c16beed558eb5d9453dbe2ba59", "stdout_sha256": "a50e97fef36830d4955926589b40cf98f73393d7ff9d201aa682adbfd2d7dafa", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 4, "acceptance_passed": true}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": [], "enforcing_mechanism": null, "enforcement_locator": null, "semantic": {"verdict": "VIOLATION_CONFIRMED", "reading": "Wide reading: the rule rules out capping inside the shared accessor at all, not merely as the default. Its stated reason is preserving the distinction between a server observation and policy.", "rationale": "The default remains unchanged, but the opt-in path keeps one accessor and applies `cap_at` inside it. That lets a policy cap alter the reported reset observation (e.g. 14400s becomes 3600s), exactly the conflation the rule rejected.", "judgements": [{"verdict": "VIOLATION_CONFIRMED", "reading": "Wide reading: the rule rules out capping inside the shared accessor at all, not merely as the default. Its stated reason is preserving the distinction between a server observation and policy.", "rationale": "The default remains unchanged, but the opt-in path keeps one accessor and applies `cap_at` inside it. That lets a policy cap alter the reported reset observation (e.g. 14400s becomes 3600s), exactly the conflation the rule rejected.", "shape": "opt-in-configurable"}, {"verdict": "VIOLATION_CONFIRMED", "reading": "Categorical reading: \u201cruled out\u201d prohibits keeping a single accessor that applies a cap internally, including on an opt-in path.", "rationale": "The patch retains one accessor and performs the cap inside it. The opt-in flag preserves the default behavior but does not avoid the explicitly ruled-out structure.", "shape": "opt-in-configurable"}], "blind_to_census_consequences": true, "why": "the blind judges agreed"}}, {"attempt_id": "replacement", "approach": "Implemented the ruled-out single capped accessor. It cannot pass existing tests: they require reporting the raw 14,400s server delay separately from the 3,600s retry cap, which this form removes. Tests were not run.", "shape": "replacement", "receipt": {"schema_version": 1, "candidate_id": "v4-00efc0041ed3118a", "attempt_id": "replacement", "repository_id": "gitseed", "registered_acceptance_command": "python3 -m pytest -q", "registered_acceptance_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "executed_command_sha256": "340b595818d9902e9103a955d6d8194afde00b522624580190cd0e97960d88a1", "command_started_at": "2026-08-23T06:01:10Z", "command_finished_at": "2026-08-23T06:01:14Z", "exit_code": 1, "test_total": 321, "test_pass": 316, "test_fail": 2, "test_skip": 3, "baseline_fingerprint": "gitseed total=321 pass=318 fail=0 skip=3 expected=", "observed_fingerprint": "gitseed total=321 pass=316 fail=2 skip=3 expected=tests/test_collect.py::TestTruncationIsReported::test_distant_reset_records_server_distance_and_cap,tests/test_collect.py::TestWaiting::test_distant_reset_sleeps_to_cap_and_reports_it", "excluded_test_ids": [], "unexpected_failures": ["tests/test_collect.py::TestTruncationIsReported::test_distant_reset_records_server_distance_and_cap", "tests/test_collect.py::TestWaiting::test_distant_reset_sleeps_to_cap_and_reports_it"], "changed_files": ["gitseed/collect/ratelimit.py", "gitseed/collect/search.py"], "sandbox_profile": "host-direct-no-agent", "runtime_identity": "Darwin arm64", "worktree_sha": "222378defcb5d2d519184b6f23146abac631faba", "final_tree_oid": "00f56db5841f6ba890ee92eb587033edd7eff60f", "stdout_sha256": "73987ccf0992788114d3fd84621d2a2bc3e678559b14e640f4bff83dbc3ae67c", "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "duration_seconds": 3, "acceptance_passed": false}, "failures_attributable_to_the_patch": [], "failures_no_implementation_can_avoid": ["tests/test_collect.py::TestTruncationIsReported::test_distant_reset_records_server_distance_and_cap", "tests/test_collect.py::TestWaiting::test_distant_reset_sleeps_to_cap_and_reports_it"], "enforcing_mechanism": "test", "enforcement_locator": "tests/test_collect.py:162 and tests/test_collect.py:279"}], "adjudication": "FUNCTIONALLY_VIOLABLE", "adjudicated_at": "2026-08-23T06:46:45Z"} diff --git a/bench/cdeb/studies/cdeb-fresh-v5/study.json b/bench/cdeb/studies/cdeb-fresh-v5/study.json index 9ca1a093..83a33876 100644 --- a/bench/cdeb/studies/cdeb-fresh-v5/study.json +++ b/bench/cdeb/studies/cdeb-fresh-v5/study.json @@ -1,21 +1,31 @@ { "study_id": "cdeb-fresh-v5", "schema_version": 1, - "phase": "stage0-authority-corpus-feasibility", + "phase": "stage1-hold", "measured_run_allowed": false, "construct": "A naturally recorded, pre-study CommitLore decision is itself a repository recorded-policy artifact.", "primary_admission_authority": "A0", "owner_testimony": "disabled", "record_id_required": false, "independent_corroboration_required": false, - "predecessors": ["cdeb-fresh-v3", "cdeb-fresh-v3r1", "cdeb-fresh-v4"], + "predecessors": [ + "cdeb-fresh-v3", + "cdeb-fresh-v3r1", + "cdeb-fresh-v4" + ], "predecessor_status": "v3 and v3r1 invalidated; v4 stage0-hold. None resumable, no measured data in any.", "predecessor_artifact_reuse": "infrastructure code only, after verification; no qualification outcome, reviewer verdict, task, gold, oracle, trajectory or result row", "product_release_tag": "v1.2.0", "product_release_commit": "90a8b212e1db70cccf69fbf48415b9c036b2d854", "product_dist_sha256": "318e16612206ae0aa3732033127b2937276ce2f142872c33a91ec04a33133b91", "study_cutoff": "2026-08-20T22:08:19Z", - "stage0_survey_repositories": ["gitseed", "agent-operator-score", "logic-pro-mcp", "agent-control-plane"], + "stage0_survey_repositories": [ + "gitseed", + "agent-operator-score", + "logic-pro-mcp", + "agent-control-plane" + ], "fixed_repository_set": null, - "created_at": "2026-08-22T01:30:00Z" + "created_at": "2026-08-22T01:30:00Z", + "verdict": "TERMINAL_HOLD" } diff --git a/test/cdeb-terminal-hardening.test.ts b/test/cdeb-terminal-hardening.test.ts index ff57398b..d0053891 100644 --- a/test/cdeb-terminal-hardening.test.ts +++ b/test/cdeb-terminal-hardening.test.ts @@ -62,10 +62,23 @@ describe('CDEB terminal hardening', () => { }); it('never resolves the terminal study as active and refuses the empty, unseeded measured-run path', () => { - expect(resolveActiveStudyRoot(CDEB_ROOT)).not.toBe(STUDY); + // Every study in this repository has now ended, so the resolver refuses + // outright rather than returning some other study. Both outcomes carry the + // claim -- the terminal study is not the one that resolves -- and the test + // accepts either instead of requiring that a live study exist. + let active: string | null = null; + try { + active = resolveActiveStudyRoot(CDEB_ROOT); + } catch (error) { + expect(String(error)).toMatch(/No active CDEB study/); + } + expect(active).not.toBe(STUDY); expect(() => assertMeasuredRunAuthorized(STUDY)).toThrow(/measured_run_allowed is not true; selection is empty; selection seed is null/); - // Whatever study is active, the measured-run gate stays shut for it too. - expect(() => assertMeasuredRunAuthorized(resolveActiveStudyRoot(CDEB_ROOT))).toThrow(/measured_run_allowed is not true/); + // And whichever study is active, if one is, the measured-run gate stays shut + // for it too. + if (active !== null) { + expect(() => assertMeasuredRunAuthorized(active as string)).toThrow(/measured_run_allowed is not true/); + } }); it('refuses caller-supplied digests that do not bind the canonical artifacts', () => { diff --git a/test/cdeb-v4-stage0-governance.test.ts b/test/cdeb-v4-stage0-governance.test.ts index cf277d87..d95428f0 100644 --- a/test/cdeb-v4-stage0-governance.test.ts +++ b/test/cdeb-v4-stage0-governance.test.ts @@ -77,10 +77,21 @@ describe('CDEB v4 Stage 0 governance', () => { }); it('has handed the active slot on and cannot take it back', () => { - // v4 reached HOLD and a successor now holds the slot. What has to stay true - // is not that v4 is active -- it is that v4 can never be active again while - // its own status says it ended. - expect(resolveActiveStudyRoot(CDEB_ROOT)).not.toBe(V4); + // v4 reached HOLD and a successor took the slot. What has to stay true is + // not that v4 is active -- it is that v4 can never be active again while its + // own status says it ended. + // + // The successor has since reached TERMINAL_HOLD too, so the slot is now + // empty and the resolver refuses outright. Both outcomes satisfy the claim, + // and the test accepts either rather than asserting that some other study + // must exist: which study holds the slot is not what this guards. + let active: string | null = null; + try { + active = resolveActiveStudyRoot(CDEB_ROOT); + } catch (error) { + expect(String(error)).toMatch(/No active CDEB study/); + } + expect(active).not.toBe(V4); const status = readJson(join(V4, 'STATUS.json')); const study = readJson(join(V4, 'study.json')); expect(status).toMatchObject({ study_id: 'cdeb-fresh-v4', phase: 'stage0-hold', measured_run_allowed: false, verdict: 'HOLD' }); diff --git a/test/cdeb-v5-governance.test.ts b/test/cdeb-v5-governance.test.ts index 45f7a751..81cfa0e9 100644 --- a/test/cdeb-v5-governance.test.ts +++ b/test/cdeb-v5-governance.test.ts @@ -48,8 +48,11 @@ describe('CDEB v5 Stage 0 governance', () => { // v4 reached a verdict rather than being invalidated. Running anything // against it would attribute the result to a study that already ended, so // the two endings are treated the same. - expect([...TERMINAL_STUDY_PHASES]).toEqual(['invalidated', 'stage0-hold']); - for (const ended of ['cdeb-fresh-v3', 'cdeb-fresh-v3r1', 'cdeb-fresh-v4']) { + // v5 later ended too, one stage further on, and stage1-hold joined the list + // for the same reason. The assertion names every ending the repository has + // reached rather than a count, so a new one has to be added deliberately. + expect([...TERMINAL_STUDY_PHASES]).toEqual(['invalidated', 'stage0-hold', 'stage1-hold']); + for (const ended of ['cdeb-fresh-v3', 'cdeb-fresh-v3r1', 'cdeb-fresh-v4', 'cdeb-fresh-v5']) { expect(() => resolveActiveStudyRoot(cdebRootNaming(ended))).toThrow( new RegExp(`Refused terminal study ${ended} as the active study`), ); @@ -73,8 +76,13 @@ describe('CDEB v5 Stage 0 governance', () => { expect(existsSync(join(CDEB_ROOT, 'studies', 'cdeb-fresh-v4', 'feasibility', 'adversarial-review.md'))).toBe(true); }); - it('resolves v5 as active with the measured run still shut', () => { - expect(resolveActiveStudyRoot(CDEB_ROOT)).toBe(V5); + it('holds v5 at TERMINAL_HOLD with the measured run never opened', () => { + // This asserted that v5 resolved as the active study. v5 has since reached + // TERMINAL_HOLD -- two of its four fixed repositories cannot produce the + // same acceptance result twice -- so the slot is empty and the resolver + // refuses. What the test is for survives the change: the measured run was + // shut when v5 was active and is shut now that it is finished. + expect(() => resolveActiveStudyRoot(CDEB_ROOT)).toThrow(/No active CDEB study/); const study = readJson(join(V5, 'study.json')); const status = readJson(join(V5, 'STATUS.json')); expect(status).toMatchObject({ study_id: 'cdeb-fresh-v5', measured_run_allowed: false }); diff --git a/test/cdeb-v5-stage1-r1.test.ts b/test/cdeb-v5-stage1-r1.test.ts index 52891ab6..52435520 100644 --- a/test/cdeb-v5-stage1-r1.test.ts +++ b/test/cdeb-v5-stage1-r1.test.ts @@ -130,6 +130,11 @@ import { type RegisteredAcceptance, } from "../bench/cdeb/freeze/acceptance-receipt-v5.ts"; import { assertFloorsUnchanged, buildCensusReport } from "../bench/cdeb/freeze/census-report-v5.ts"; +import { + TERMINAL_STUDY_PHASES, + assertStudyNotTerminal, + resolveActiveStudyRoot, +} from "../bench/cdeb/active-study.ts"; import { MIN_NEEDS, assertNeedScoutAnswer, @@ -2249,6 +2254,42 @@ describe("the census artifacts are generated, not maintained", () => { }); }); +describe("the study is finished and cannot be named active again", () => { + it("refuses stage1-hold as an active study", () => { + // v4 stopped at stage0-hold and the phase list learned that word. v5 stopped + // one stage later, and a list that only knew where v4 stopped would have let + // v5 keep resolving as active with a published verdict already in its tree. + expect(TERMINAL_STUDY_PHASES).toContain("stage1-hold"); + expect(() => { + assertStudyNotTerminal(V5, "cdeb-fresh-v5"); + }).toThrow(/its phase is stage1-hold/); + }); + + it("declares no active study rather than naming a successor that does not exist", () => { + const declaration = readJson(join(V5, "..", "..", "ACTIVE-STUDY.json")); + expect(declaration.status).toBe("no-active-study"); + expect(declaration.active_study_id).toBe(null); + expect(declaration.last_terminal_study_id).toBe("cdeb-fresh-v5"); + // The status word and the id have to agree, and the resolver refuses a + // declaration where they do not. + expect(() => { + resolveActiveStudyRoot(join(V5, "..", "..")); + }).toThrow(/No active CDEB study/); + }); + + it("carries the verdict and its basis in the study's own status", () => { + const status = readJson(join(V5, "STATUS.json")); + expect(status).toMatchObject({ + study_id: "cdeb-fresh-v5", + phase: "stage1-hold", + measured_run_allowed: false, + verdict: "TERMINAL_HOLD", + successor_required: true, + }); + expect(String(status.verdict_basis)).toMatch(/two of the four fixed repositories/); + }); +}); + describe("ยง19.15 the implementation executes no episode", () => { it("declares the state plainly and leaves the randomization schedule uncomputable", () => { const randomization = readJson(join(R1, "randomization-plan.json"));