Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 19 additions & 55 deletions .github/workflows/world-sim.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
name: World Sim

# Plays the deterministic scenario book (`workbench/sim-world`) against the
# runtime in this commit, once per log world, and publishes the two summaries.
# runtime in this commit and publishes its summary.
#
# This lane never blocks a merge, by design. Three scenarios in the book fail on
# purpose: each one is a reproduction of a corruption the runtime can still
# produce, stating the outcome its own durable log implies, and staying red
# until the runtime gets there. A gate that goes red on every PR is a gate
# everyone learns to ignore, so the job publishes numbers instead of verdicts —
# and the number to watch is in the comment, not the check mark.
#
# mint-ordered (production): 38 passed, 3 failed, 3 violations
# append-only: 41 passed, 0 failed, 0 violations
#
# A fourth red is a regression. Two means something got fixed and a scenario
# is ready to retire. The append-only column is the measurement the pair exists
# for: it says which of the three would close if event positions were assigned
# at commit instead of at the handler's mint.
# This lane never blocks a merge, by design. The job publishes its numbers
# instead of a verdict, so the comment is the result to watch, not the check
# mark.

on:
push:
Expand Down Expand Up @@ -49,8 +38,8 @@ jobs:
# on a PR it has nothing to say about. The step summary and the artifact
# still carry whatever did happen.
continue-on-error: true
# The book itself is ~11s per world on a laptop. Everything else here is
# install and build.
# The book itself is ~11s on a laptop. Everything else here is install and
# build.
timeout-minutes: 15
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
Expand All @@ -72,35 +61,17 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-workflow-dev

# Both worlds run with their real exit codes: `continue-on-error` records
# the non-zero without ending the job, so the step's own status still
# says whether the book was clean. Paths are absolute because pnpm runs
# the script from the package directory, not the workspace root.
- name: Play the book (mint-ordered log)
id: mint
continue-on-error: true
run: |
pnpm --filter @workflow/sim-world-workbench sim \
--no-color \
--title 'Mint-ordered log' \
--summary-file "${{ github.workspace }}/world-sim-mint.md" \
--detail-file "${{ github.workspace }}/world-sim-mint.txt"

- name: Play the book (append-only log)
id: append-only
- name: Play the book
id: world-sim
continue-on-error: true
run: |
pnpm --filter @workflow/sim-world-workbench sim \
--no-color \
--append-only \
--title 'Append-only log' \
--summary-file "${{ github.workspace }}/world-sim-append-only.md" \
--detail-file "${{ github.workspace }}/world-sim-append-only.txt"
--summary-file "${{ github.workspace }}/world-sim.md" \
--detail-file "${{ github.workspace }}/world-sim.txt"

# Four visible lines when collapsed: the heading, one line saying what
# this is, and one per world. Everything else is behind a fold. The
# comment is reposted on every push to the PR, so what it costs when it
# has nothing new to say is the thing to keep small.
# The comment is reposted on every push to the PR, so what it costs when
# it has nothing new to say is the thing to keep small.
- name: Render summary
if: always()
run: |
Expand All @@ -110,22 +81,17 @@ jobs:
echo
echo "Simulated world deterministic testing for races. [Traces]($run_url)"
echo
for world in mint append-only; do
if [ -f "world-sim-$world.md" ]; then
if [ -f world-sim.md ]; then
# The summary names its detail file by the path it was given,
# which is absolute so that pnpm's package-directory cwd cannot
# scatter them. Strip the workspace prefix back off, leaving
# the bare name the artifact below actually contains.
sed "s|${{ github.workspace }}/||g" "world-sim-$world.md"
# Blank line between the two folds. Adjacent HTML blocks with
# nothing between them get parsed as one, and the second world
# disappears into the first one's fold.
sed "s|${{ github.workspace }}/||g" world-sim.md
echo
else
echo "🟠 _The $world run produced no summary — see the job log._"
else
echo "🟠 _The run produced no summary — see the job log._"
echo
fi
done
fi
} | tee world-sim-summary.md >> "$GITHUB_STEP_SUMMARY"

# Skipped on forks, where `pull_request` grants read-only permissions and
Expand All @@ -147,9 +113,7 @@ jobs:
name: world-sim-traces
path: |
world-sim-summary.md
world-sim-mint.md
world-sim-mint.txt
world-sim-append-only.md
world-sim-append-only.txt
world-sim.md
world-sim.txt
retention-days: 7
if-no-files-found: ignore
62 changes: 15 additions & 47 deletions packages/world-sim/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,18 +121,7 @@ every *validation* is kept, because rejections are the observable contract.

## World behaviors

A scenario picks the world it plays in, and each behavior below changes a rule
the runtime is written against.

**Mint-ordered log** — the default. A position is assigned when the event's
handler mints its id, and the event is committed to storage separately. The id
*is* the log's sort key, so a write held between the two lands *behind* events
minted later and committed sooner: an event can arrive in the past, and a read
taken in between saw a log the log itself went on to contradict.

**Append-only log** (`appendOnlyLog: true`) — a position is assigned at commit.
A write overtaken while it was held gives up its position and re-takes the tail.
Two things follow:
**Event log** — positions are assigned at commit. Two things follow:

- Log order is commit order. Nothing is inserted behind a row a reader has
already seen, so no two reads can disagree about the past.
Expand All @@ -141,12 +130,8 @@ Two things follow:
into lag, and lag is what an optimistic-concurrency fence can see; a hole is
what it cannot.

Uncontended writes are untouched either way: a position that is still the newest
when it commits keeps its id, so a scenario that never holds a write mid-flight
produces a byte-identical log in both. `withholdNextEvent` follows the same
rule — a hole in the mint-ordered log, a truncated tail under append-only —
which is why `StaleRead` reports `{ eventId, hidden, truncated }` and the trace
distinguishes a lagging read from a stale one.
`withholdNextEvent` models a lagging replica by truncating the visible tail;
`StaleRead` reports `{ eventId, hidden, truncated }` for that read.

**Precondition fence** (`preconditionGuard: true`) — rejects a write whose
snapshot is strictly older than the newest externally originated event. It is a
Expand All @@ -166,11 +151,8 @@ client's own — newest loaded position, and how many loaded events sit at or
below it. A write the facade attached no snapshot to did not come from a replay
context and is never fenced.

Each is a spec field, and `RunScenarioOptions` carries a run-wide override —
`pnpm sim --append-only`, `--fence` / `--no-fence` — where `undefined` leaves
each scenario's own choice alone. Playing one book under two behaviors and
diffing the results is what the pair is for; [DESIGN.md
§5](./DESIGN.md#the-two-guards) has the guards in full.
Each is a spec field, and `RunScenarioOptions` carries a run-wide override for
the fence. [DESIGN.md §5](./DESIGN.md#the-two-guards) has the guards in full.

## Usage

Expand Down Expand Up @@ -377,12 +359,8 @@ const script: ScenarioScript = async (sim) => {
`{stepName, token, correlationId, where, label, timeoutMs}`.

Both advances hold a writer whose event has no position yet, so a write that
commits during the hold sorts *ahead* of it. For the other order — an event that
already owns an earlier slot and has not appeared — hold the write itself with
[`sim.beginHookDelivery`](#withholdings), which reserves the position and hands
back a `commit()`. Under `appendOnlyLog` that reservation is provisional: an
overtaken write gives it up and re-takes the tail, which is exactly how the world
closes the gap. See [World behaviors](#world-behaviors).
commits during the hold sorts ahead of it. `sim.beginHookDelivery` can defer an
external hook write until its `commit()` call.

`runTo` is **level-triggered**: it consults recorded history, so a point this
writer already passed is an `AlreadyPassedError` naming the point rather than a
Expand Down Expand Up @@ -435,17 +413,16 @@ and changes what storage answers.
| method | writer | description |
| --- | --- | --- |
| `sim.withholdNextEvent(reads?)` | whichever commits next | Hide the next event committed to storage from the next `reads` event-log reads (default 1). Call it immediately before the write to hide. |
| `sim.beginHookDelivery(token, payload)` | `external` | Deliver a hook, withheld between its two halves: assigned a position in the event log, not committed to storage. Returns `{eventId, commit()}`. |
| `sim.beginHookDelivery(token, payload)` | `external` | Begin an external hook delivery and return `commit()`, which writes it at the log tail. |

`beginHookDelivery` is the one place inside an `external` writer a script can
reach, and it is a withholding rather than an advance because holding that
writer would be the wrong model: an out-of-band receiver is a separate process,
so nothing of the run's is blocked while its write is in flight. Holding an
inline write would stall the delivery that made it, and the reader with it.

Both change shape with the log. Under `appendOnlyLog` a withheld read is cut
short at the withheld event instead of missing it from the middle — the log can
be behind, never wrong — and an overtaken hook re-takes the tail on `commit()`.
A withheld read is cut short at the withheld event: the log can be behind, never
wrong.

### Everything else a script can do

Expand All @@ -457,7 +434,6 @@ be behind, never wrong — and an overtaken hook re-takes the tail on `commit()`
| `deliverQueued(select?)` | Deliver one queued message now, concurrently with a held writer |
| `note(msg)` / `check(name, cond)` | Record a marker / an assertion in the trace; a false check fails the scenario |
| `world` | Read-only snapshot: runs, events, steps, hooks, waits, pending messages, rejected calls |
| `appendOnlyLog` | Which log this run is playing against — for *phrasing* a check, never for branching the tempo |

A scenario with no script at all is a control: the run plays out on the default
schedule, and the only question is whether the log it leaves reproduces it.
Expand Down Expand Up @@ -530,22 +506,14 @@ so adding an option to one of them is not a change to the package's public
signature. Promote a name to the entry when something outside the package needs
it, not before.

**A new world flag is tri-state at the runner.** `ScenarioSpec` carries the
scenario's own choice, `RunScenarioOptions` the run-wide override, and
`undefined` means "leave it to the spec" — not the same as `false`, because a
scenario that asked for the flag must keep it. `run.ts` maps `--x` / `--no-x`
onto that, and the resolved value reaches `createSimWorld` and the chips line.

**Anything a scenario can observe has to survive replay.** `verifyReplay`
re-plays the log in a fresh world built from the same options, so a store rule
that is not applied there turns every scenario using it red for the wrong
reason.
re-plays the log in a fresh world, so a store rule that is not applied there
turns every scenario using it red for the wrong reason.

**Tests come in two shapes.** `src/*.test.ts` are vitest units against the
pieces in isolation — copy `store.test.ts` for anything that changes what the
log looks like, where the append-only block is written as pairs asserting
*opposite* outcomes in the two worlds. The scenario book is the integration
test; run it before and after and diff the counts.
pieces in isolation — copy `store.test.ts` for anything that changes what the
log looks like. The scenario book is the integration test; run it before and
after and diff the counts.

## What this does *not* give you

Expand Down
23 changes: 0 additions & 23 deletions packages/world-sim/src/invariants.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,29 +133,6 @@ describe('invariants', () => {
).toContain('log.monotonic-order');
});

it('skips the order rule when the world makes no such promise', () => {
const a = event({
eventType: 'run_created',
eventData: {
deploymentId: 'd',
workflowName: 'w',
input: new Uint8Array(),
},
});
const b = event({ eventType: 'run_started' });
expect(
rules(
checkInvariants({
runId: RUN,
events: [a, b],
runs: [run('running')],
steps: [],
waits: [],
})
)
).not.toContain('log.monotonic-order');
});

it('catches an out-of-order log', () => {
const a = event({
eventType: 'run_created',
Expand Down
20 changes: 4 additions & 16 deletions packages/world-sim/src/invariants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,8 @@ export interface InvariantInput {
runId: string;
/** The run's events in log order — the order every reader sees them in. */
events: Event[];
/**
* The same events in the order they were *committed*, supplied only by a
* world that promises the two orders agree — i.e. an append-only log.
*
* Only `log.monotonic-order` reads it, and it has to: comparing the sorted
* array against sort order can only ever pass, which is why that rule was
* unfirable before. Under a mint-ordered log the field is omitted and the
* rule is skipped, because there an out-of-order commit is the premise the
* scenario deliberately injected — production mints ids at the handler
* boundary, so its log gains rows in the past by design. Asserting otherwise
* would fail every scenario that holds a write across a peer's commit, which
* is the setup, not the fault.
*/
eventsInCommitOrder?: Event[];
/** The same events in the order they were committed. */
eventsInCommitOrder: Event[];
runs: WorkflowRun[];
steps: Step[];
waits: Wait[];
Expand Down Expand Up @@ -78,13 +66,13 @@ export function checkInvariants(input: InvariantInput): InvariantViolation[] {
}

// `events.list` sorts by (createdAt, eventId), and replay consumes events in
// that order. An append-only log promises commit order *is* that order; if it
// that order. Commit order must be that order; if it
// is not, the log gained a row behind a position readers had already passed,
// so a read taken in between saw a sequence the finished log contradicts.
// Walking the sorted array could never notice — it is sorted, so it is
// monotonic by construction. This is the check that the promise was kept.
let previousKey = '';
for (const event of input.eventsInCommitOrder ?? []) {
for (const event of input.eventsInCommitOrder) {
Comment thread
vercel[bot] marked this conversation as resolved.
const key = `${event.createdAt.toISOString()}|${event.eventId}`;
if (previousKey && key <= previousKey) {
add(
Expand Down
13 changes: 1 addition & 12 deletions packages/world-sim/src/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,6 @@ export interface ReplayCheckInput {
events: readonly Event[];
handler: (req: Request) => Promise<Response>;
limits: Required<ScenarioLimits>;
/**
* Replay under the same log rules as the run being checked. It cannot change
* the outcome — the replay seeds a finished log and writes only at a clock
* past its tail, so nothing it appends can be overtaken — but a replay
* playing by different rules than the run it verifies is a trap worth not
* setting.
*/
appendOnlyLog?: boolean;
}

export interface ReplayCheckResult {
Expand Down Expand Up @@ -133,10 +125,7 @@ export async function verifyReplay(
const replayClock = createVirtualClock(terminal.createdAt.getTime() + 1);
const uninstallClock = replayClock.install();

const replayWorld = createSimWorld({
clock: replayClock,
appendOnlyLog: input.appendOnlyLog,
});
const replayWorld = createSimWorld({ clock: replayClock });
replayWorld.store.seedFromLog(seeded);
replayWorld.registerHandler(WORKFLOW_QUEUE_PREFIX, handler);
replayWorld.setScenarioApi(() => {
Expand Down
5 changes: 1 addition & 4 deletions packages/world-sim/src/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -414,10 +414,7 @@ export function renderScenario(
` run=${result.runId || '(none)'} outcome=${result.outcome} ` +
`events=${result.events.length} deliveries=${result.deliveries} ` +
`worldCalls=${result.worldCalls} virtual=${formatDuration(result.virtualElapsedMs)} ` +
`wall=${result.wallMs.toFixed(0)}ms replay=${describeReplay(result)}` +
// Only when it is on. The default is production, and a line that
// repeats "this is the ordinary world" on every scenario says nothing.
(result.appendOnlyLog ? ' log=append-only' : ''),
`wall=${result.wallMs.toFixed(0)}ms replay=${describeReplay(result)}`,
'dim'
)
);
Expand Down
Loading
Loading