From 03f77526b29e1a9a0b338b624071c581ebb417dd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 31 Oct 2026 02:28:58 -0700 Subject: [PATCH 1/2] fix(execution): name the signal that killed a sandbox process in its error --- .../remote-sandbox/code-failure.test.ts | 28 +++++++++++++- .../execution/remote-sandbox/code-failure.ts | 37 ++++++++++++++++++- .../remote-sandbox/conformance.test.ts | 12 ++++++ .../sim/lib/execution/remote-sandbox/index.ts | 7 +++- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts b/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts index 64b59629886..445e86c3555 100644 --- a/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { processCodeFailure } from '@/lib/execution/remote-sandbox/code-failure' +import { + describeProcessExit, + processCodeFailure, +} from '@/lib/execution/remote-sandbox/code-failure' /** Node 22's crash report when `fetch` loses its socket before any response bytes. */ const FETCH_SOCKET_CLOSED = `node:internal/deps/undici/undici:13510 @@ -196,6 +199,29 @@ describe('processCodeFailure', () => { expect(error).toEqual({ name, value, traceback: stderr }) }) + it('names the signal for a process killed after printing only a warning', () => { + const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.' + const killed = processCodeFailure({ stdout: '', stderr: warning, exitCode: 137 }) + const exited = processCodeFailure({ stdout: '', stderr: warning, exitCode: 1 }) + + expect(killed.error).toEqual({ + name: 'Error', + value: 'Process exited with code 137 (killed by SIGKILL, usually out of memory)', + traceback: warning, + }) + expect(exited.error?.name).toBe('Warning') + }) + + it.each([ + [1, 'Process exited with code 1'], + [128, 'Process exited with code 128'], + [129, 'Process exited with code 129 (killed by signal 1)'], + [143, 'Process exited with code 143 (killed by SIGTERM)'], + [193, 'Process exited with code 193'], + ])('describes exit code %i', (exitCode, description) => { + expect(describeProcessExit(exitCode)).toBe(description) + }) + it('reads the traceback from stdout when stderr is empty', () => { const { error } = processCodeFailure({ stdout: PYTHON_TRACEBACK, stderr: '', exitCode: 1 }) diff --git a/apps/sim/lib/execution/remote-sandbox/code-failure.ts b/apps/sim/lib/execution/remote-sandbox/code-failure.ts index 990a4120f16..38faf394d74 100644 --- a/apps/sim/lib/execution/remote-sandbox/code-failure.ts +++ b/apps/sim/lib/execution/remote-sandbox/code-failure.ts @@ -14,12 +14,47 @@ import type { SandboxCodeResult, SandboxCommandResult } from '@/lib/execution/re const ERROR_HEADLINE = /^(?:\[(?:[A-Za-z_$][\w.$]*)?(?:Error|Exception)|\[?[A-Za-z_$][\w.$]* \[[\w.$-]+\]|[A-Za-z_$][\w.$]*):(?:\s|$)/ +const SIGNAL_NAMES: Readonly> = { + 6: 'SIGABRT', + 9: 'SIGKILL', + 11: 'SIGSEGV', + 15: 'SIGTERM', +} + +/** + * `Process exited with code N`, naming the signal behind a code above 128. A killed process often + * printed only a warning first, and that warning must not read as the reason it failed; SIGKILL + * (137) is usually the out-of-memory killer. + */ +export function describeProcessExit(exitCode: number): string { + const described = `Process exited with code ${exitCode}` + const signal = killingSignal(exitCode) + if (signal === undefined) return described + const name = SIGNAL_NAMES[signal] ?? `signal ${signal}` + return `${described} (killed by ${name}${signal === 9 ? ', usually out of memory' : ''})` +} + +/** The signal a shell's `128 + n` exit code reports, if the code is one. */ +function killingSignal(exitCode: number): number | undefined { + return exitCode > 128 && exitCode <= 128 + 64 ? exitCode - 128 : undefined +} + /** * Converts a failed code command into a code result, naming the error from the - * last error headline of the traceback (stderr, else stdout). + * last error headline of the traceback (stderr, else stdout). A process a signal + * killed raised nothing: anything it printed (a warning) is not the cause, so the + * signal names the error and the output stays as the traceback. */ export function processCodeFailure(result: SandboxCommandResult): SandboxCodeResult { const traceback = result.stderr || result.stdout + if (killingSignal(result.exitCode) !== undefined) { + return { + text: '', + stdout: result.stdout, + stderr: result.stderr, + error: { name: 'Error', value: describeProcessExit(result.exitCode), traceback }, + } + } const headline = traceback .split('\n') .reverse() diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index d62d6db26c5..960cfed2513 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -1261,6 +1261,18 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { }) }) + it("leads a killed command's error with its signal, not the warning it printed first", async () => { + const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.' + stubShellCommand(provider, provider === 'daytona' ? warning : '', warning, 137) + + const res = await executeShellInSandbox({ code: 'python train.py', envs: {}, timeoutMs: 1000 }) + + expect(res.error).toBe( + `Process exited with code 137 (killed by SIGKILL, usually out of memory): ${warning}` + ) + expect(res.stdout).toContain(warning) + }) + it('terminates shell execution when streamed process output exceeds the byte budget', async () => { const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1) if (provider === 'e2b') { diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index abf685496c1..a16a464c294 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -11,6 +11,7 @@ import { import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics' import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime' import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server' +import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure' import { type CreatedSandbox, createSandbox, @@ -1152,8 +1153,10 @@ async function executeShellInSandboxWithinBudget( if (result.exitCode !== 0) { // Daytona merges both streams into stdout (stderr is always empty), so fall // back to stdout for the real command output before the generic message. - const errorMessage = - result.stderr || result.stdout || `Process exited with code ${result.exitCode}` + const output = result.stderr || result.stdout + const errorMessage = output + ? `${describeProcessExit(result.exitCode)}: ${output}` + : describeProcessExit(result.exitCode) logger.error('Sandbox shell execution error', { sandboxId, exitCode: result.exitCode, From e020a2b7557179b53a821d73a87be31fea3a8717 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 31 Oct 2026 03:01:37 -0700 Subject: [PATCH 2/2] fix(execution): describe a signal's exit code without claiming a kill --- .../remote-sandbox/code-failure.test.ts | 17 +++++--- .../execution/remote-sandbox/code-failure.ts | 42 +++++++++---------- .../remote-sandbox/conformance.test.ts | 4 +- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts b/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts index 445e86c3555..97fcce0344c 100644 --- a/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/code-failure.test.ts @@ -199,24 +199,29 @@ describe('processCodeFailure', () => { expect(error).toEqual({ name, value, traceback: stderr }) }) - it('names the signal for a process killed after printing only a warning', () => { + it('keeps the printed headline and adds the signal exit code beside it', () => { const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.' const killed = processCodeFailure({ stdout: '', stderr: warning, exitCode: 137 }) const exited = processCodeFailure({ stdout: '', stderr: warning, exitCode: 1 }) expect(killed.error).toEqual({ - name: 'Error', - value: 'Process exited with code 137 (killed by SIGKILL, usually out of memory)', + name: 'Warning', + value: + 'You are sending unauthenticated requests to the HF Hub. [Process exited with code 137 (the exit code of SIGKILL, usually out of memory)]', + traceback: warning, + }) + expect(exited.error).toEqual({ + name: 'Warning', + value: 'You are sending unauthenticated requests to the HF Hub.', traceback: warning, }) - expect(exited.error?.name).toBe('Warning') }) it.each([ [1, 'Process exited with code 1'], [128, 'Process exited with code 128'], - [129, 'Process exited with code 129 (killed by signal 1)'], - [143, 'Process exited with code 143 (killed by SIGTERM)'], + [129, 'Process exited with code 129 (the exit code of signal 1)'], + [143, 'Process exited with code 143 (the exit code of SIGTERM)'], [193, 'Process exited with code 193'], ])('describes exit code %i', (exitCode, description) => { expect(describeProcessExit(exitCode)).toBe(description) diff --git a/apps/sim/lib/execution/remote-sandbox/code-failure.ts b/apps/sim/lib/execution/remote-sandbox/code-failure.ts index 38faf394d74..9159fb7336e 100644 --- a/apps/sim/lib/execution/remote-sandbox/code-failure.ts +++ b/apps/sim/lib/execution/remote-sandbox/code-failure.ts @@ -22,39 +22,32 @@ const SIGNAL_NAMES: Readonly> = { } /** - * `Process exited with code N`, naming the signal behind a code above 128. A killed process often - * printed only a warning first, and that warning must not read as the reason it failed; SIGKILL - * (137) is usually the out-of-memory killer. + * `Process exited with code N`, naming the signal a code above 128 stands for. A process a signal + * killed often printed only a warning first, so the code must be visible beside whatever it + * printed. The code alone cannot prove a kill (a program may exit 137 itself), so it is described + * as the signal's code; SIGKILL's (137) usually means the out-of-memory killer. */ export function describeProcessExit(exitCode: number): string { - const described = `Process exited with code ${exitCode}` - const signal = killingSignal(exitCode) - if (signal === undefined) return described - const name = SIGNAL_NAMES[signal] ?? `signal ${signal}` - return `${described} (killed by ${name}${signal === 9 ? ', usually out of memory' : ''})` + const note = signalCodeNote(exitCode) + return `Process exited with code ${exitCode}${note ? ` (${note})` : ''}` } -/** The signal a shell's `128 + n` exit code reports, if the code is one. */ -function killingSignal(exitCode: number): number | undefined { - return exitCode > 128 && exitCode <= 128 + 64 ? exitCode - 128 : undefined +/** What a shell's `128 + n` exit code stands for, if the code is one. */ +function signalCodeNote(exitCode: number): string | undefined { + if (exitCode <= 128 || exitCode > 128 + 64) return undefined + const signal = exitCode - 128 + const name = SIGNAL_NAMES[signal] ?? `signal ${signal}` + return `the exit code of ${name}${signal === 9 ? ', usually out of memory' : ''}` } /** * Converts a failed code command into a code result, naming the error from the - * last error headline of the traceback (stderr, else stdout). A process a signal - * killed raised nothing: anything it printed (a warning) is not the cause, so the - * signal names the error and the output stays as the traceback. + * last error headline of the traceback (stderr, else stdout). A signal's exit code + * is appended to the value, since a killed process's last printed line (often a + * warning) is not necessarily why it stopped. */ export function processCodeFailure(result: SandboxCommandResult): SandboxCodeResult { const traceback = result.stderr || result.stdout - if (killingSignal(result.exitCode) !== undefined) { - return { - text: '', - stdout: result.stdout, - stderr: result.stderr, - error: { name: 'Error', value: describeProcessExit(result.exitCode), traceback }, - } - } const headline = traceback .split('\n') .reverse() @@ -66,10 +59,13 @@ export function processCodeFailure(result: SandboxCommandResult): SandboxCodeRes const separator = errorLine?.indexOf(':') ?? -1 const parsedErrorLine = errorLine ?? '' const name = separator > 0 ? parsedErrorLine.slice(0, separator) : 'Error' - const value = + const headlineValue = separator > 0 ? parsedErrorLine.slice(separator + 1).trim() : parsedErrorLine || 'Execution failed' + const value = signalCodeNote(result.exitCode) + ? `${headlineValue} [${describeProcessExit(result.exitCode)}]` + : headlineValue return { text: '', stdout: result.stdout, diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 960cfed2513..a4f8e5bacd4 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -1261,14 +1261,14 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { }) }) - it("leads a killed command's error with its signal, not the warning it printed first", async () => { + it("leads a failed command's error with its exit code and the signal it stands for", async () => { const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.' stubShellCommand(provider, provider === 'daytona' ? warning : '', warning, 137) const res = await executeShellInSandbox({ code: 'python train.py', envs: {}, timeoutMs: 1000 }) expect(res.error).toBe( - `Process exited with code 137 (killed by SIGKILL, usually out of memory): ${warning}` + `Process exited with code 137 (the exit code of SIGKILL, usually out of memory): ${warning}` ) expect(res.stdout).toContain(warning) })