Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ const FAILURE_CODE_REASONS = {
assistant_error: 'assistant_failed',
missing_assistant_reply: 'assistant_no_reply',
payment_required: 'billing',
kilo_output_limit: 'assistant_failed',
user_interrupt: 'user_cancelled',
container_shutdown: 'container_shutdown',
system_interrupt: 'interrupted',
Expand Down
1 change: 1 addition & 0 deletions packages/db/src/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5428,6 +5428,7 @@ export type CloudAgentSessionRunFailureCode =
| 'wrapper_error_after_activity'
| 'missing_assistant_reply'
| 'payment_required'
| 'kilo_output_limit'
| 'user_interrupt'
| 'container_shutdown'
| 'system_interrupt'
Expand Down
12 changes: 12 additions & 0 deletions packages/worker-utils/src/cloud-agent-failure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@ describe('classifyCloudAgentFailure', () => {
});
});

it.each([
'kilo_output_limit',
'wrapper_no_output',
'wrapper_ping_timeout',
'wrapper_disconnected',
] as const)('classifies %s as platform wrapper_liveness', code => {
expect(classifyCloudAgentFailure({ source: 'run', stage: 'agent_activity', code })).toEqual({
responsibility: 'platform',
reason: 'wrapper_liveness',
});
});

it('preserves ambiguous assistant and source-control failures as unknown', () => {
expect(
classifyCloudAgentFailure({
Expand Down
2 changes: 2 additions & 0 deletions packages/worker-utils/src/cloud-agent-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export const CLOUD_AGENT_FAILURE_CODES = [
'wrapper_error_after_activity',
'missing_assistant_reply',
'payment_required',
'kilo_output_limit',
'user_interrupt',
'container_shutdown',
'system_interrupt',
Expand Down Expand Up @@ -234,6 +235,7 @@ export function classifyCloudAgentFailure(
case 'wrapper_error_before_activity':
case 'wrapper_error_after_activity':
case 'missing_assistant_reply':
case 'kilo_output_limit':
return classified('platform', 'wrapper_liveness');
case 'assistant_error':
case 'payment_required':
Expand Down
5 changes: 5 additions & 0 deletions packages/worker-utils/src/cloud-agent-queue-report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,15 @@ export const CloudAgentRunFailureClassifications = [
{ failureStage: 'post_dispatch_no_activity', failureCode: 'missing_assistant_reply' },
{ failureStage: 'post_dispatch_no_activity', failureCode: 'payment_required' },
{ failureStage: 'post_dispatch_no_activity', failureCode: 'model_missing' },
{ failureStage: 'post_dispatch_no_activity', failureCode: 'kilo_output_limit' },
{ failureStage: 'agent_activity', failureCode: 'assistant_error' },
{ failureStage: 'agent_activity', failureCode: 'payment_required' },
{ failureStage: 'agent_activity', failureCode: 'model_missing' },
{ failureStage: 'agent_activity', failureCode: 'wrapper_error_after_activity' },
{ failureStage: 'agent_activity', failureCode: 'wrapper_no_output' },
{ failureStage: 'agent_activity', failureCode: 'wrapper_ping_timeout' },
{ failureStage: 'agent_activity', failureCode: 'wrapper_disconnected' },
{ failureStage: 'agent_activity', failureCode: 'kilo_output_limit' },
{ failureStage: 'interruption', failureCode: 'user_interrupt' },
{ failureStage: 'interruption', failureCode: 'container_shutdown' },
{ failureStage: 'interruption', failureCode: 'system_interrupt' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ const GENERIC_FAILURE_MESSAGES = {
wrapper_error_after_activity: 'Agent wrapper failed while processing the message',
missing_assistant_reply: 'No assistant reply was produced',
payment_required: 'Assistant request failed: insufficient credits',
kilo_output_limit: 'Assistant response hit the output length limit',
user_interrupt: 'The message was interrupted by the user',
container_shutdown: 'The agent container shut down',
system_interrupt: 'The message was interrupted',
Expand Down
64 changes: 64 additions & 0 deletions services/cloud-agent-next/src/session/wrapper-supervisor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,27 @@ describe('WrapperSupervisor', () => {
expect(harness.events.map(event => event.streamEventType)).toEqual(['cloud.message.failed']);
});

it('preserves wrapper_no_output after agent activity', async () => {
const acceptedAt = 2_000;
const noOutputDeadlineAt = acceptedAt + WRAPPER_NO_OUTPUT_TIMEOUT_MS;
const harness = createHarness([
liveRuntimeState({ noOutputDeadlineAt, nextPingAt: noOutputDeadlineAt + 1 }),
OWNED_WRAPPER_LEASE,
]);
await putSessionMessageState(harness.storage, {
...acceptedMessage(),
agentActivityObservedAt: acceptedAt + 1_000,
});

await harness.supervisor.runMaintenance(noOutputDeadlineAt);

await expect(getSessionMessageState(harness.storage, MESSAGE_ID)).resolves.toMatchObject({
status: 'failed',
failureStage: 'agent_activity',
failureCode: 'wrapper_no_output',
});
});

it('terminates an unresponsive wrapper on ping timeout before no-output expires', async () => {
const pingDeadlineAt = 92_000;
const noOutputDeadlineAt = 332_000;
Expand All @@ -989,6 +1010,49 @@ describe('WrapperSupervisor', () => {
});
});

it('preserves wrapper_ping_timeout after agent activity', async () => {
const pingDeadlineAt = 92_000;
const noOutputDeadlineAt = 332_000;
const harness = createHarness([
liveRuntimeState({ pingDeadlineAt, noOutputDeadlineAt }),
OWNED_WRAPPER_LEASE,
]);
await putSessionMessageState(harness.storage, {
...acceptedMessage(),
agentActivityObservedAt: 9_000,
});

await harness.supervisor.runMaintenance(pingDeadlineAt);

await expect(getSessionMessageState(harness.storage, MESSAGE_ID)).resolves.toMatchObject({
status: 'failed',
failureStage: 'agent_activity',
failureCode: 'wrapper_ping_timeout',
});
});

it('preserves kilo_output_limit after agent activity', async () => {
const harness = createHarness([liveRuntimeState(), OWNED_WRAPPER_LEASE]);
await putSessionMessageState(harness.storage, {
...acceptedMessage(),
agentActivityObservedAt: 9_000,
});

await harness.supervisor.onTerminalEvent({
wrapperRunId: WRAPPER_RUN_ID,
status: 'failed',
errorSource: 'assistant',
failureCode: 'kilo_output_limit',
error: 'Assistant response hit the output length limit',
});

await expect(getSessionMessageState(harness.storage, MESSAGE_ID)).resolves.toMatchObject({
status: 'failed',
failureStage: 'agent_activity',
failureCode: 'kilo_output_limit',
});
});

it('defers liveness failure while disconnect grace is active for the current connection', async () => {
const pingDeadlineAt = 92_000;
const noOutputDeadlineAt = 332_000;
Expand Down
21 changes: 14 additions & 7 deletions services/cloud-agent-next/src/session/wrapper-supervisor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type { MessageSettlementOutbox } from './message-settlement-outbox.js';
import {
classifyAssistantFailure,
classifyAssistantFailureMessage,
genericFailureMessage,
} from './safe-failure-projection.js';
import { countPendingSessionMessages, type SessionQueueStorage } from './pending-messages.js';
import type { SessionMessageQueue } from './session-message-queue.js';
Expand Down Expand Up @@ -649,7 +650,7 @@ export function createWrapperSupervisor(
error,
completionSource: 'wrapper_failure',
failureStage: activityObserved ? 'agent_activity' : 'post_dispatch_no_activity',
failureCode: activityObserved ? 'wrapper_error_after_activity' : failureCode,
failureCode,
});
}
await messageSettlementOutbox.releaseWrapperTerminalWaitForIdleBatch();
Expand Down Expand Up @@ -728,7 +729,7 @@ export function createWrapperSupervisor(
error: 'Wrapper disconnected',
completionSource: 'wrapper_failure',
failureStage: activityObserved ? 'agent_activity' : 'post_dispatch_no_activity',
failureCode: activityObserved ? 'wrapper_error_after_activity' : 'wrapper_disconnected',
failureCode: 'wrapper_disconnected',
});
}
await clearWrapperRuntimeIdentity(
Expand Down Expand Up @@ -1431,17 +1432,23 @@ export function createWrapperSupervisor(
if (status === 'failed') {
if (errorSource === 'assistant') {
const assistantFailure = classifyAssistantFailure(error);
const failureCode =
terminalFailureCode ?? assistantFailure.terminalCode ?? 'assistant_error';
const usesExplicitTerminalCode = terminalFailureCode === 'kilo_output_limit';
await messageSettlementOutbox.terminalizeSessionMessageOnce(message.messageId, {
kind: 'failed',
reason: 'assistant_error',
error: error ?? 'Assistant request failed',
completionSource: 'wrapper_failure',
failureStage: 'agent_activity',
failureCode:
terminalFailureCode ?? assistantFailure.terminalCode ?? 'assistant_error',
assistantFailureReason: assistantFailure.reason,
providerOwnership: assistantFailure.providerOwnership,
safeFailureMessage: assistantFailure.safeMessage,
failureCode,
...(usesExplicitTerminalCode
? { safeFailureMessage: genericFailureMessage(failureCode) }
: {
assistantFailureReason: assistantFailure.reason,
providerOwnership: assistantFailure.providerOwnership,
safeFailureMessage: assistantFailure.safeMessage,
}),
...(persistedModelNotFoundDiagnostics
? { modelNotFoundRuntimeDiagnostics: persistedModelNotFoundDiagnostics }
: {}),
Expand Down
6 changes: 5 additions & 1 deletion services/cloud-agent-next/src/shared/protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ export type IngestEvent = {
data: unknown;
};

export const WrapperTerminalFailureCodes = ['payment_required', 'model_missing'] as const;
export const WrapperTerminalFailureCodes = [
'payment_required',
'model_missing',
'kilo_output_limit',
] as const;
export type WrapperTerminalFailureCode = (typeof WrapperTerminalFailureCodes)[number];

/**
Expand Down
1 change: 1 addition & 0 deletions services/cloud-agent-next/src/telemetry/queue-reports.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const FAILED_RUN_DIAGNOSTIC_MESSAGES: Partial<
wrapper_error_after_activity: 'Wrapper failed after agent activity',
missing_assistant_reply: 'No assistant reply was produced',
payment_required: 'Model request failed: insufficient credits',
kilo_output_limit: 'Assistant response hit the output length limit',
unclassified: 'Run failed without a classified cause',
};

Expand Down
79 changes: 57 additions & 22 deletions services/cloud-agent-next/src/websocket/ingest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1233,30 +1233,65 @@ describe('createIngestHandler', () => {
});

it.each([
{ failureCode: 'payment_required' as const, error: 'Insufficient credits' },
{ failureCode: 'model_missing' as const, error: 'Model not found' },
])('forwards $failureCode wrapper failures to the session coordinator', async failure => {
const doContext = createNewPathDOContext();
const handler = createIngestHandler(
createFakeState(),
createFakeEventQueries(),
SESSION_ID,
vi.fn(),
doContext
);
const ws = createFakeWebSocket(makeNewPathAttachment());
{
failureCode: 'payment_required' as const,
error: 'Insufficient credits',
safeMessage: 'Assistant request failed: insufficient credits',
},
{
failureCode: 'model_missing' as const,
error: 'Model not found',
safeMessage: 'Assistant request failed: model not found',
},
{
failureCode: 'kilo_output_limit' as const,
error: 'Assistant response hit the output length limit',
safeMessage: 'Assistant response hit the output length limit',
},
])(
'forwards $failureCode wrapper failures to the session coordinator',
async ({ safeMessage, ...failure }) => {
const doContext = createNewPathDOContext();
const eventQueries = createFakeEventQueries();
const broadcast = vi.fn();
const handler = createIngestHandler(
createFakeState(),
eventQueries,
SESSION_ID,
broadcast,
doContext
);
const ws = createFakeWebSocket(makeNewPathAttachment());

await handler.handleIngestMessage(
ws,
makeStreamMessage('error', { fatal: true, ...failure })
);
await handler.handleIngestMessage(
ws,
makeStreamMessage('error', { fatal: true, errorSource: 'assistant', ...failure })
);

expect(doContext.handleWrapperTerminalEvent).toHaveBeenCalledWith({
wrapperRunId: WRAPPER_RUN_ID,
status: 'failed',
...failure,
});
});
expect(doContext.handleWrapperTerminalEvent).toHaveBeenCalledWith({
wrapperRunId: WRAPPER_RUN_ID,
status: 'failed',
errorSource: 'assistant',
...failure,
});
expect(broadcast).toHaveBeenCalledWith(
expect.objectContaining({
stream_event_type: 'cloud.status',
payload: JSON.stringify({ cloudStatus: { type: 'error', message: safeMessage } }),
})
);
expect(eventQueries.insert).toHaveBeenCalledWith(
expect.objectContaining({
payload: JSON.stringify({
fatal: true,
errorSource: 'assistant',
error: safeMessage,
message: safeMessage,
}),
})
);
}
);

it('does NOT terminalize on wrapper complete event (new path)', async () => {
const state = createFakeState();
Expand Down
22 changes: 20 additions & 2 deletions services/cloud-agent-next/src/websocket/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
} from '../session/ingest-handlers/index.js';
import {
WrapperTerminalFailureCodes,
type WrapperTerminalFailureCode,
type CompleteEventData,
type KilocodeEventData,
type CloudStatusData,
Expand All @@ -37,6 +38,7 @@ import type { TerminalizeParams } from '../session/session-message-state.js';
import {
classifyAssistantFailure,
classifyAssistantFailureMessage,
genericFailureMessage,
} from '../session/safe-failure-projection.js';
import { parseModelNotFoundRuntimeDiagnostics } from '../shared/runtime-model-diagnostics.js';
import {
Expand Down Expand Up @@ -153,6 +155,22 @@ function sanitizeKilocodeEventData(data: unknown): unknown {
return data;
}

/**
* The wrapper sends fixed text for kilo_output_limit that text classification
* cannot recognize, so resolve it from the structured code.
* payment_required/model_missing keep text classification: the model-not-found
* diagnostics gate keys off the classified safe message.
*/
function safeAssistantFailureMessage(
failureCode: WrapperTerminalFailureCode | undefined,
rawError: unknown
): string {
if (failureCode === 'kilo_output_limit') {
return genericFailureMessage(failureCode);
}
return classifyAssistantFailureMessage(rawError);
}

function sanitizePublicEventData(eventType: string, data: unknown): unknown {
if (eventType === 'kilocode') return sanitizeKilocodeEventData(data);

Expand All @@ -162,7 +180,7 @@ function sanitizePublicEventData(eventType: string, data: unknown): unknown {
const rawError = parsed.data.error ?? parsed.data.message;
const safeMessage =
parsed.data.errorSource === 'assistant'
? classifyAssistantFailureMessage(rawError)
? safeAssistantFailureMessage(parsed.data.failureCode, rawError)
: 'Agent wrapper failed';
return {
fatal: parsed.data.fatal,
Expand Down Expand Up @@ -923,7 +941,7 @@ export function createIngestHandler(
const fatalMessage = errorData.error ?? errorData.message ?? 'Fatal error';
const safeFatalMessage =
errorData.errorSource === 'assistant'
? classifyAssistantFailureMessage(fatalMessage)
? safeAssistantFailureMessage(errorData.failureCode, fatalMessage)
: 'Agent wrapper failed';
const shouldForwardModelNotFoundDiagnostics =
errorData.errorSource === 'assistant' &&
Expand Down
Loading