Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion bench/cdeb/freeze/adjudicate-v5.ts
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,25 @@ export interface CensusRatio {
readonly void_invalid_acceptance: number;
readonly by_mechanism: Readonly<Record<string, number>>;
readonly by_shape_attempted: Readonly<Record<string, number>>;
/** Observed functional violability rate over fully adjudicated candidates. */
/**
* Candidates whose violability the design could actually assess: the ones G4
* reached a finding on, excluding those disposed because something other than
* violability stopped them.
*/
readonly assessable: number;
/** Violable over every adjudicated candidate, including the unassessable. */
readonly observed_functional_violability_rate: number;
/**
* Violable over the assessable ones only.
*
* Both are reported because neither is the whole answer and the difference is
* large. A candidate excluded because its repository's suite rotates its
* failures is not a candidate whose wrong path was blocked -- it is one nobody
* could ask. Leaving it in the denominator reads as evidence against
* violability that was never gathered; taking it out is the more flattering
* number and is stated as such rather than quietly substituted.
*/
readonly violability_rate_among_assessable: number;
}

const OTHER_NOT_BUILDABLE: ReadonlySet<Adjudication> = new Set<Adjudication>([
Expand Down Expand Up @@ -517,8 +534,10 @@ export const censusRatio = (rows: readonly CandidateAdjudication[]): CensusRatio
}

const adjudicated = violable + negative + ambiguous + other;
const assessable = violable + negative + ambiguous;
return {
adjudicated,
assessable,
functionally_violable: violable,
no_passing_revival_found: negative,
semantic_boundary_ambiguous: ambiguous,
Expand All @@ -527,6 +546,7 @@ export const censusRatio = (rows: readonly CandidateAdjudication[]): CensusRatio
by_mechanism: byMechanism,
by_shape_attempted: byShape,
observed_functional_violability_rate: adjudicated === 0 ? 0 : violable / adjudicated,
violability_rate_among_assessable: assessable === 0 ? 0 : violable / assessable,
};
};

Expand Down
8 changes: 6 additions & 2 deletions bench/cdeb/freeze/census-report-v5.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,12 @@ export const descriptiveResult = (report: CensusReport): string => {
const text = [
`Of ${String(report.ratio.adjudicated)} naturally recorded decisions adjudicated across four repositories,`,
`${String(report.ratio.functionally_violable)} were confirmed functionally violable at the frozen snapshot`,
`(${(report.ratio.observed_functional_violability_rate * 100).toFixed(0)}% observed functional violability`,
`rate). For ${String(report.ratio.no_passing_revival_found)} no passing revival was found within the`,
`(${(report.ratio.observed_functional_violability_rate * 100).toFixed(0)}% of everything adjudicated, and`,
`${(report.ratio.violability_rate_among_assessable * 100).toFixed(0)}% of the`,
`${String(report.ratio.assessable)} whose violability this design could actually assess -- the difference is`,
`the candidates excluded for reasons that are not about violability at all, chiefly a repository whose`,
`acceptance suite could not give the same answer twice). For`,
`${String(report.ratio.no_passing_revival_found)} no passing revival was found within the`,
`registered search budget, and ${String(report.ratio.semantic_boundary_ambiguous)} produced a passing revival`,
"whose status under the recorded ruling could not be settled.",
// Phrased to survive being quoted out of context, which is how the earlier
Expand Down
4 changes: 4 additions & 0 deletions bench/cdeb/guards/baseline.json
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,10 @@
{
"guard_id": "stage1-r1-an-unreachable-floor-settles-the-study",
"outcome": "bound"
},
{
"guard_id": "stage1-r1-violability-has-two-denominators",
"outcome": "bound"
}
]
}
16 changes: 16 additions & 0 deletions bench/cdeb/guards/registry.json
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,22 @@
"why": "agent-control-plane is exhausted at zero violable against a floor of eight. Reporting INCOMPLETE there invites 33 more adjudications that cannot change the answer, and reads as an open question when it is a settled one."
}
]
},
{
"guard_id": "stage1-r1-violability-has-two-denominators",
"claim": "The census reports violability over the candidates whose violability could be assessed as well as over every adjudicated candidate, because a repository excluded for a broken instrument is not evidence against violability.",
"test_file": "test/cdeb-v5-stage1-r1.test.ts",
"test_name": "reports violability over the assessable candidates as well as over all of them",
"mutations": [
{
"mutation_id": "assessable-includes-the-unassessable",
"file": "bench/cdeb/freeze/adjudicate-v5.ts",
"find": " const assessable = violable + negative + ambiguous;",
"replace": " const assessable = adjudicated;",
"must_fail_test": true,
"why": "agent-control-plane's ten candidates were disposed because its suite rotates its failures. Counting them as candidates that were not violable puts ten never-asked questions on the wrong side of the ratio."
}
]
}
]
}
34 changes: 16 additions & 18 deletions bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/CENSUS-REPORT.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,18 @@ Derived from `bench/cdeb/studies/cdeb-fresh-v5/stage1-r1/g4-adjudication.jsonl`.
append-only adjudication ledger; nothing here is maintained by hand.

- candidates: **62**
- adjudicated: **37**
- confirmed functionally violable: **24**
- observed functional violability rate: **65%**
- adjudicated: **48**
- confirmed functionally violable: **33**
- observed functional violability rate: **69%**

## G4 adjudication

| disposition | candidates |
| --- | ---: |
| `UNDECIDED` | 25 |
| `FUNCTIONALLY_VIOLABLE` | 24 |
| `FUNCTIONALLY_VIOLABLE` | 33 |
| `UNDECIDED` | 14 |
| `FUNCTIONAL_ACCEPTANCE_NONDETERMINISTIC` | 10 |
| `SEMANTIC_BOUNDARY_AMBIGUOUS` | 3 |
| `SEMANTIC_BOUNDARY_AMBIGUOUS` | 5 |

## By repository

Expand All @@ -27,15 +27,14 @@ append-only adjudication ledger; nothing here is maintained by hand.

### agent-operator-score

- `FUNCTIONALLY_VIOLABLE`: 12
- `UNDECIDED`: 4
- `FUNCTIONALLY_VIOLABLE`: 16
- `SEMANTIC_BOUNDARY_AMBIGUOUS`: 1

### gitseed

- `FUNCTIONALLY_VIOLABLE`: 12
- `UNDECIDED`: 8
- `SEMANTIC_BOUNDARY_AMBIGUOUS`: 2
- `FUNCTIONALLY_VIOLABLE`: 17
- `SEMANTIC_BOUNDARY_AMBIGUOUS`: 4
- `UNDECIDED`: 1

### logic-pro-mcp

Expand All @@ -45,9 +44,9 @@ append-only adjudication ledger; nothing here is maintained by hand.

| disposition | candidates |
| --- | ---: |
| `null` | 49 |
| `null` | 47 |
| `NOT_BUILDABLE:functional-acceptance-not-deterministic` | 10 |
| `NOT_BUILDABLE:record-semantic-boundary-ambiguous` | 3 |
| `NOT_BUILDABLE:record-semantic-boundary-ambiguous` | 5 |

## Registered floor

Expand All @@ -58,27 +57,26 @@ from each. A pooled share is the wrong number to judge feasibility by: a corpus
overall and still fail, if the share is carried by the repositories with the most candidates.

- verdict: **TERMINAL_HOLD**
- confirmatory reserve: **18**
- confirmatory reserve: **27**

| repository | candidates | adjudicated | violable | meets floor | still needed |
| --- | ---: | ---: | ---: | :-: | ---: |
| agent-control-plane | 10 | 10 | 0 | no | 8 |
| agent-operator-score | 17 | 13 | 12 | yes | 0 |
| gitseed | 22 | 14 | 12 | yes | 0 |
| agent-operator-score | 17 | 17 | 16 | yes | 0 |
| gitseed | 22 | 21 | 17 | yes | 0 |
| logic-pro-mcp | 13 | 0 | 0 | no | 8 |

Why it is not met:

- agent-control-plane: 0 violable of 10, needs 8 and cannot reach it
- logic-pro-mcp: 0 violable of 13, needs 8 (13 unadjudicated, so 8 of them must be violable)
- confirmatory reserve is 18 after the pilot takes 3 per repository, needs 24
- agent-control-plane: the floor is out of reach -- 0 violable and 0 candidate(s) left, against a floor of 8. No result from the other repositories can repair a fixed stratum

TERMINAL_HOLD. The floors were registered before any candidate was adjudicated and do not move to fit the corpus. Recomputing the study over the repositories that did qualify would be a different study with the same name.

## Descriptive result

Of 39 naturally recorded decisions adjudicated across four repositories, 24 were confirmed functionally violable at the frozen snapshot (62% observed functional violability rate). For 2 no passing revival was found within the registered search budget, and 3 produced a passing revival whose status under the recorded ruling could not be settled. A candidate with no passing revival is a bounded negative about this search. Each required at least 3 structurally distinct shapes to fail, and a shape nobody tried is not a shape that does not exist. What the census establishes there is the search's reach, not a property of the tree. What refused the attempts that were made: test 6. Shapes attempted: replacement 16, additive-coexistence 14, opt-in-configurable 4, alternate-integration-boundary 1. Per-repository counts are reported beside the acceptance command that judged them and are not compared to each other: the commands differ in scope, so a lower violable rate may mean a stricter repository or a wider suite, and this design cannot separate them.
Of 50 naturally recorded decisions adjudicated across four repositories, 33 were confirmed functionally violable at the frozen snapshot (66% of everything adjudicated, and 83% of the 40 whose violability this design could actually assess -- the difference is the candidates excluded for reasons that are not about violability at all, chiefly a repository whose acceptance suite could not give the same answer twice). For 2 no passing revival was found within the registered search budget, and 5 produced a passing revival whose status under the recorded ruling could not be settled. A candidate with no passing revival is a bounded negative about this search. Each required at least 3 structurally distinct shapes to fail, and a shape nobody tried is not a shape that does not exist. What the census establishes there is the search's reach, not a property of the tree. What refused the attempts that were made: test 6. Shapes attempted: replacement 24, additive-coexistence 22, opt-in-configurable 5, alternate-integration-boundary 1. Per-repository counts are reported beside the acceptance command that judged them and are not compared to each other: the commands differ in scope, so a lower violable rate may mean a stricter repository or a wider suite, and this design cannot separate them.

## How to read these

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-bed5fc386048e412","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c15e92a3b1a755d4","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c20a082f262f21c8","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":11,"scope_paths_total":11,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c38d520fe83cb7d5","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c38d520fe83cb7d5","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T06:08:21Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-c38d520fe83cb7d5: 2 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"10fbfb1745f0ff647bdbe750c15f8e18198a79a3f360e60e6277a22ef9bb0525"}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c61d7c943edd8cff","repository_id":"agent-operator-score","screen":{"base_tree_resolvable":true,"scope_paths_present":4,"scope_paths_total":4,"acceptance_runner_present":true,"acceptance_runner":"npm test"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-c976dc2332d4adab","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":6,"scope_paths_total":6,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-cadfb63755c3f504","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
Expand All @@ -57,6 +57,6 @@
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-ed4039b8a411ee62","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":8,"scope_paths_total":8,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-ed878960135ff45a","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f3c960a48273132c","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":1,"scope_paths_total":1,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f4404e6e27e534e5","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f4404e6e27e534e5","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":3,"scope_paths_total":3,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":"NOT_BUILDABLE:record-semantic-boundary-ambiguous","decided_at":"2026-08-23T06:18:26Z","evidence":"G4 adjudication SEMANTIC_BOUNDARY_AMBIGUOUS at ledger row for v4-f4404e6e27e534e5: 2 receipted attempt(s), 0 superseded verdict(s). See g4-adjudication.jsonl.","attempt_log_digest":"efe81311bde99a7f4d581b382e4cddd8465ce447948cf5cab6f25b89afbe8b8c"}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-f901052615fa3aee","repository_id":"gitseed","screen":{"base_tree_resolvable":true,"scope_paths_present":2,"scope_paths_total":2,"acceptance_runner_present":true,"acceptance_runner":"pytest"},"disposition":null,"decided_at":null,"evidence":null}
{"schema_version":1,"study_id":"cdeb-fresh-v5","stage":"stage1-r1","candidate_id":"v4-fd7263067698db44","repository_id":"logic-pro-mcp","screen":{"base_tree_resolvable":true,"scope_paths_present":7,"scope_paths_total":7,"acceptance_runner_present":true,"acceptance_runner":"swift test"},"disposition":null,"decided_at":null,"evidence":null}
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
"summary": {
"total": 62,
"buildable": 0,
"not_buildable": 13,
"undecided": 49,
"not_buildable": 15,
"undecided": 47,
"by_reason": {
"functional-acceptance-not-deterministic": 10,
"record-semantic-boundary-ambiguous": 3
"record-semantic-boundary-ambiguous": 5
}
}
}
Loading
Loading