Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion apps/sim/lib/execution/remote-sandbox/code-failure.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { describe, expect, it } from 'vitest'
import { processCodeFailure } from '@/lib/execution/remote-sandbox/code-failure'
import {
describeProcessExit,
processCodeFailure,
} from '@/lib/execution/remote-sandbox/code-failure'

/** Node 22's crash report when `fetch` loses its socket before any response bytes. */
const FETCH_SOCKET_CLOSED = `node:internal/deps/undici/undici:13510
Expand Down Expand Up @@ -196,6 +199,34 @@ describe('processCodeFailure', () => {
expect(error).toEqual({ name, value, traceback: stderr })
})

it('keeps the printed headline and adds the signal exit code beside it', () => {
const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
const killed = processCodeFailure({ stdout: '', stderr: warning, exitCode: 137 })
const exited = processCodeFailure({ stdout: '', stderr: warning, exitCode: 1 })

expect(killed.error).toEqual({
name: 'Warning',
value:
'You are sending unauthenticated requests to the HF Hub. [Process exited with code 137 (the exit code of SIGKILL, usually out of memory)]',
traceback: warning,
})
expect(exited.error).toEqual({
name: 'Warning',
value: 'You are sending unauthenticated requests to the HF Hub.',
traceback: warning,
})
})

it.each([
[1, 'Process exited with code 1'],
[128, 'Process exited with code 128'],
[129, 'Process exited with code 129 (the exit code of signal 1)'],
[143, 'Process exited with code 143 (the exit code of SIGTERM)'],
[193, 'Process exited with code 193'],
])('describes exit code %i', (exitCode, description) => {
expect(describeProcessExit(exitCode)).toBe(description)
})

it('reads the traceback from stdout when stderr is empty', () => {
const { error } = processCodeFailure({ stdout: PYTHON_TRACEBACK, stderr: '', exitCode: 1 })

Expand Down
35 changes: 33 additions & 2 deletions apps/sim/lib/execution/remote-sandbox/code-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,37 @@ import type { SandboxCodeResult, SandboxCommandResult } from '@/lib/execution/re
const ERROR_HEADLINE =
/^(?:\[(?:[A-Za-z_$][\w.$]*)?(?:Error|Exception)|\[?[A-Za-z_$][\w.$]* \[[\w.$-]+\]|[A-Za-z_$][\w.$]*):(?:\s|$)/

const SIGNAL_NAMES: Readonly<Record<number, string>> = {
6: 'SIGABRT',
9: 'SIGKILL',
11: 'SIGSEGV',
15: 'SIGTERM',
}

/**
* `Process exited with code N`, naming the signal a code above 128 stands for. A process a signal
* killed often printed only a warning first, so the code must be visible beside whatever it
* printed. The code alone cannot prove a kill (a program may exit 137 itself), so it is described
* as the signal's code; SIGKILL's (137) usually means the out-of-memory killer.
*/
export function describeProcessExit(exitCode: number): string {
const note = signalCodeNote(exitCode)
return `Process exited with code ${exitCode}${note ? ` (${note})` : ''}`
}

/** What a shell's `128 + n` exit code stands for, if the code is one. */
function signalCodeNote(exitCode: number): string | undefined {
if (exitCode <= 128 || exitCode > 128 + 64) return undefined
const signal = exitCode - 128
const name = SIGNAL_NAMES[signal] ?? `signal ${signal}`
return `the exit code of ${name}${signal === 9 ? ', usually out of memory' : ''}`
}

/**
* Converts a failed code command into a code result, naming the error from the
* last error headline of the traceback (stderr, else stdout).
* last error headline of the traceback (stderr, else stdout). A signal's exit code
* is appended to the value, since a killed process's last printed line (often a
* warning) is not necessarily why it stopped.
*/
export function processCodeFailure(result: SandboxCommandResult): SandboxCodeResult {
const traceback = result.stderr || result.stdout
Expand All @@ -31,10 +59,13 @@ export function processCodeFailure(result: SandboxCommandResult): SandboxCodeRes
const separator = errorLine?.indexOf(':') ?? -1
const parsedErrorLine = errorLine ?? ''
const name = separator > 0 ? parsedErrorLine.slice(0, separator) : 'Error'
const value =
const headlineValue =
separator > 0
? parsedErrorLine.slice(separator + 1).trim()
: parsedErrorLine || 'Execution failed'
const value = signalCodeNote(result.exitCode)
? `${headlineValue} [${describeProcessExit(result.exitCode)}]`
: headlineValue
return {
text: '',
stdout: result.stdout,
Expand Down
12 changes: 12 additions & 0 deletions apps/sim/lib/execution/remote-sandbox/conformance.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1261,6 +1261,18 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
})
})

it("leads a failed command's error with its exit code and the signal it stands for", async () => {
const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
stubShellCommand(provider, provider === 'daytona' ? warning : '', warning, 137)

const res = await executeShellInSandbox({ code: 'python train.py', envs: {}, timeoutMs: 1000 })

expect(res.error).toBe(
`Process exited with code 137 (the exit code of SIGKILL, usually out of memory): ${warning}`
)
expect(res.stdout).toContain(warning)
})

it('terminates shell execution when streamed process output exceeds the byte budget', async () => {
const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1)
if (provider === 'e2b') {
Expand Down
7 changes: 5 additions & 2 deletions apps/sim/lib/execution/remote-sandbox/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics'
import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime'
import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server'
import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure'
import {
type CreatedSandbox,
createSandbox,
Expand Down Expand Up @@ -1152,8 +1153,10 @@ async function executeShellInSandboxWithinBudget(
if (result.exitCode !== 0) {
// Daytona merges both streams into stdout (stderr is always empty), so fall
// back to stdout for the real command output before the generic message.
const errorMessage =
result.stderr || result.stdout || `Process exited with code ${result.exitCode}`
const output = result.stderr || result.stdout
const errorMessage = output
? `${describeProcessExit(result.exitCode)}: ${output}`
: describeProcessExit(result.exitCode)
logger.error('Sandbox shell execution error', {
sandboxId,
exitCode: result.exitCode,
Expand Down
Loading