From cdba4c31203fe2bdb8cdffe06e960ac37e2e77bc Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Tue, 18 Aug 2026 15:34:48 -0700 Subject: [PATCH 01/15] chore: trigger CI to verify NUT failure logs [do-not-merge] --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 69129dc5..53b78b6c 100644 --- a/README.md +++ b/README.md @@ -2498,3 +2498,5 @@ EXAMPLES _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/2.2.3/src/commands/agent/validate/authoring-bundle.ts)_ + + From 8b6f429077ff7704f4a70217d996bee89db56891 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Mon, 19 Oct 2026 14:13:47 -0700 Subject: [PATCH 02/15] test(nuts): collapse redundant run-eval invocations to reduce auth churn [do-not-merge] The run-eval NUTs issued the identical JSON-payload command 4x and the identical YAML-spec command 2x, each a fresh CLI process doing a fresh OAuth token refresh against the single shared scratch-org admin user. That burst trips Salesforce's per-user login-rate limit (login rate exceeded), the dominant NUT failure. Run each distinct command once in a before() hook and share the parsed result across the tests that assert against it. All assertions preserved; genuinely-distinct commands untouched. --- test/nuts/agent.test.run-eval.nut.ts | 80 +++++++++++++++------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 8c73b42c..3533be5b 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -29,55 +29,65 @@ describe('agent test run-eval', function () { const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); + // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's + // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the + // parsed result across every test that asserts against it, so a rate-limit failure surfaces + // once rather than per-test. + let jsonResult: RunEvalResult; + before(async function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); + + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; }); describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; - expect(result.jsonOutput?.result.summary.passed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.failed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.scored).to.be.a('number'); - expect(result.jsonOutput?.result.summary.errors).to.be.a('number'); + expect(jsonResult).to.be.ok; + expect(jsonResult.tests).to.be.an('array'); + expect(jsonResult.tests.length).to.be.greaterThan(0); + expect(jsonResult.summary).to.be.ok; + expect(jsonResult.summary.passed).to.be.a('number'); + expect(jsonResult.summary.failed).to.be.a('number'); + expect(jsonResult.summary.scored).to.be.a('number'); + expect(jsonResult.summary.errors).to.be.a('number'); }); it('should run evaluation with normalized payload', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests[0]).to.be.ok; - expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing'); - expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); - expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array'); + expect(jsonResult.tests[0]).to.be.ok; + expect(jsonResult.tests[0].id).to.equal('test-topic-routing'); + expect(jsonResult.tests[0].status).to.be.oneOf(['passed', 'failed']); + expect(jsonResult.tests[0].evaluations).to.be.an('array'); }); }); describe('run-eval with YAML file', () => { - it('should run evaluation with YAML test spec file', async () => { + // The YAML-spec run-eval is run once and shared across the two tests that assert against it, + // for the same OAuth-rate-limit reason as the JSON-payload run above. + let yamlResult: RunEvalResult; + + before(() => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); + const output = execCmd(command, { ensureExitCode: 0 }); + expect(output.jsonOutput?.result).to.be.ok; + yamlResult = output.jsonOutput!.result; + }); - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; + it('should run evaluation with YAML test spec file', async () => { + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); + expect(yamlResult.tests.length).to.be.greaterThan(0); + expect(yamlResult.summary).to.be.ok; }); it('should auto-infer agent name from YAML subjectName', async () => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - // Should succeed without explicit --api-name flag - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); }); it('should handle YAML spec with contextVariables', async () => { @@ -173,11 +183,8 @@ describe('agent test run-eval', function () { describe('run-eval output structure', () => { it('should include test summaries with correct structure', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests).to.be.an('array'); - const firstTest = result.jsonOutput?.result.tests[0]; + expect(jsonResult.tests).to.be.an('array'); + const firstTest = jsonResult.tests[0]; expect(firstTest).to.have.property('id'); expect(firstTest).to.have.property('status'); expect(firstTest).to.have.property('evaluations'); @@ -187,10 +194,7 @@ describe('agent test run-eval', function () { }); it('should include summary with all metrics', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - const summary = result.jsonOutput?.result.summary; + const summary = jsonResult.summary; expect(summary).to.have.property('passed'); expect(summary).to.have.property('failed'); expect(summary).to.have.property('scored'); From c2bb5f93d126563cea096b32c0d0d482076d0821 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 10:57:12 -0700 Subject: [PATCH 03/15] test(nuts): isolate run-eval on a dedicated scratch-org user [do-not-merge] Give the agent test run-eval NUT its own scratch-org user (System Administrator profile + EinsteinGPTPromptTemplateManager, mirroring the shared admin) so its OAuth grant/session churn no longer concentrates on the shared admin and trips the per-user login-rate cap. --- test/nuts/agent.test.run-eval.nut.ts | 24 +++++++-------- test/nuts/shared-setup.ts | 45 +++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 3533be5b..c6569bae 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername } from './shared-setup.js'; +import { getTestSession, getRunEvalUsername } from './shared-setup.js'; describe('agent test run-eval', function () { // Increase timeout for setup since shared setup includes long waits and deployments @@ -39,7 +39,7 @@ describe('agent test run-eval', function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); expect(jsonExec.jsonOutput?.result).to.be.ok; jsonResult = jsonExec.jsonOutput!.result; @@ -71,7 +71,7 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(() => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; @@ -91,7 +91,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -104,7 +104,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -112,7 +112,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -121,20 +121,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -157,7 +157,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -169,7 +169,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index f3b9176c..97c0b67d 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -17,7 +17,7 @@ import { join } from 'node:path'; import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; -import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; +import { Org, SfError, User, DefaultUserFields, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ @@ -26,6 +26,9 @@ import { sleep, ensureArray } from '@salesforce/kit'; let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; +// Dedicated user for the `agent test run-eval` NUT, isolating its OAuth grant/session churn +// from the shared admin so the full suite's load doesn't trip the per-user login-rate cap. +let runEvalUsername: string | undefined; // Non-sensitive scratch-org identifiers captured at setup for CI failure correlation. let orgId: string | undefined; let orgPod: string | undefined; @@ -129,6 +132,30 @@ export async function getTestSession(): Promise { await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); + // Create a dedicated user (in this same scratch org) for the `agent test run-eval` NUT. + // Running that heavy NUT's commands against a separate user isolates its OAuth + // grant/session churn from the shared admin, so the full suite's load no longer evicts + // the admin's sessions and trips the per-user login-rate cap. Mirror the admin's access: + // System Administrator profile + the same permission set, so run-eval behaves identically. + console.log('Creating dedicated run-eval user...'); + const adminUsername = defaultOrg.username; + const runEvalFields = (await DefaultUserFields.create({ templateUser: adminUsername })).getFields(); + // DefaultUserFields defaults to the Standard User profile; override to System Administrator + // so the dedicated user has admin-equivalent access for run-eval. + const sysAdminProfile = await connection.singleRecordQuery<{ Id: string }>( + "SELECT Id FROM Profile WHERE Name='System Administrator'" + ); + runEvalFields.profileId = sysAdminProfile.Id; + const runEvalUser = await User.create({ org }); + const runEvalAuthInfo = await runEvalUser.createUser(runEvalFields); + runEvalUsername = runEvalAuthInfo.getUsername(); + const runEvalUserId = runEvalAuthInfo.getFields().userId; + if (!runEvalUsername || !runEvalUserId) { + throw new Error('Failed to create run-eval user: no username/id returned'); + } + await runEvalUser.assignPermissionSets(runEvalUserId, ['EinsteinGPTPromptTemplateManager']); + console.log(`[NUT-CONTEXT] runEvalUsername=${runEvalUsername}`); + // Create agent user using the new CLI command console.log('Creating agent user...'); const agentUserResult = execCmd<{ username: string }>( @@ -280,6 +307,22 @@ export function getUsername(): string { return testSession.orgs.get('default')!.username!; } +/** + * Gets the dedicated run-eval user's username. This user lives in the same shared scratch org + * and is created once during getTestSession(), so that `agent test run-eval`'s OAuth grant/session + * churn is isolated from the shared admin user. + * Throws an error if the session (and thus the user) hasn't been created yet. + */ +export function getRunEvalUsername(): string { + if (!testSession) { + throw new Error('Test session not available. Call getTestSession() first.'); + } + if (!runEvalUsername) { + throw new Error('Run-eval user not available. Call getTestSession() first.'); + } + return runEvalUsername; +} + /** * Gets the agent user, username, from the shared test session. * Throws an error if the session hasn't been created yet. From 07e9b2bc982f5ebfac92e63e023d0bb3b737d388 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 14:21:18 -0700 Subject: [PATCH 04/15] test(nuts): revert dedicated run-eval user; add token-safe auth trace [do-not-merge] Dedicated-user experiment (c2bb5f9) did not clear login-rate: run-eval's first grant on a brand-new user still tripped the cap, proving it is not per-user (org- or IP-wide). Revert that change and instead run the run-eval JSON hook under SF_LOG_LEVEL=trace, dumping the token-redacted OAuth-exchange lines to CI to capture the rate-limit response's HTTP status / endpoint / Retry-After and settle per-org vs per-IP. --- test/nuts/agent.test.run-eval.nut.ts | 40 ++++++++----- test/nuts/shared-setup.ts | 88 +++++++++++++++------------- 2 files changed, 71 insertions(+), 57 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index c6569bae..63080acf 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getRunEvalUsername } from './shared-setup.js'; +import { getTestSession, getUsername, dumpCliAuthTrace } from './shared-setup.js'; describe('agent test run-eval', function () { // Increase timeout for setup since shared setup includes long waits and deployments @@ -39,10 +39,20 @@ describe('agent test run-eval', function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; - const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); - expect(jsonExec.jsonOutput?.result).to.be.ok; - jsonResult = jsonExec.jsonOutput!.result; + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + // DIAGNOSTIC (do-not-merge): run under SF_LOG_LEVEL=trace so the CLI's own (token-redacted) + // log records the OAuth token exchange; dump the relevant lines regardless of pass/fail so CI + // shows the `login rate exceeded` response's HTTP status/endpoint/Retry-After (per-org vs per-IP). + try { + const jsonExec = execCmd(jsonCommand, { + ensureExitCode: 0, + env: { ...process.env, SF_LOG_LEVEL: 'trace' }, + }); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; + } finally { + dumpCliAuthTrace('run-eval-json-before'); + } }); describe('run-eval with JSON file', () => { @@ -71,7 +81,7 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(() => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; @@ -91,7 +101,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -104,7 +114,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -112,7 +122,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -121,20 +131,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -157,7 +167,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -169,7 +179,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 97c0b67d..dfed61bf 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,9 +15,10 @@ */ import { join } from 'node:path'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; -import { Org, SfError, User, DefaultUserFields, Lifecycle, Connection } from '@salesforce/core'; +import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ @@ -26,9 +27,6 @@ import { sleep, ensureArray } from '@salesforce/kit'; let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; -// Dedicated user for the `agent test run-eval` NUT, isolating its OAuth grant/session churn -// from the shared admin so the full suite's load doesn't trip the per-user login-rate cap. -let runEvalUsername: string | undefined; // Non-sensitive scratch-org identifiers captured at setup for CI failure correlation. let orgId: string | undefined; let orgPod: string | undefined; @@ -132,30 +130,6 @@ export async function getTestSession(): Promise { await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); - // Create a dedicated user (in this same scratch org) for the `agent test run-eval` NUT. - // Running that heavy NUT's commands against a separate user isolates its OAuth - // grant/session churn from the shared admin, so the full suite's load no longer evicts - // the admin's sessions and trips the per-user login-rate cap. Mirror the admin's access: - // System Administrator profile + the same permission set, so run-eval behaves identically. - console.log('Creating dedicated run-eval user...'); - const adminUsername = defaultOrg.username; - const runEvalFields = (await DefaultUserFields.create({ templateUser: adminUsername })).getFields(); - // DefaultUserFields defaults to the Standard User profile; override to System Administrator - // so the dedicated user has admin-equivalent access for run-eval. - const sysAdminProfile = await connection.singleRecordQuery<{ Id: string }>( - "SELECT Id FROM Profile WHERE Name='System Administrator'" - ); - runEvalFields.profileId = sysAdminProfile.Id; - const runEvalUser = await User.create({ org }); - const runEvalAuthInfo = await runEvalUser.createUser(runEvalFields); - runEvalUsername = runEvalAuthInfo.getUsername(); - const runEvalUserId = runEvalAuthInfo.getFields().userId; - if (!runEvalUsername || !runEvalUserId) { - throw new Error('Failed to create run-eval user: no username/id returned'); - } - await runEvalUser.assignPermissionSets(runEvalUserId, ['EinsteinGPTPromptTemplateManager']); - console.log(`[NUT-CONTEXT] runEvalUsername=${runEvalUsername}`); - // Create agent user using the new CLI command console.log('Creating agent user...'); const agentUserResult = execCmd<{ username: string }>( @@ -308,30 +282,60 @@ export function getUsername(): string { } /** - * Gets the dedicated run-eval user's username. This user lives in the same shared scratch org - * and is created once during getTestSession(), so that `agent test run-eval`'s OAuth grant/session - * churn is isolated from the shared admin user. - * Throws an error if the session (and thus the user) hasn't been created yet. + * Gets the agent user, username, from the shared test session. + * Throws an error if the session hasn't been created yet. */ -export function getRunEvalUsername(): string { +export function getAgentUsername(): string | undefined { if (!testSession) { throw new Error('Test session not available. Call getTestSession() first.'); } - if (!runEvalUsername) { - throw new Error('Run-eval user not available. Call getTestSession() first.'); - } - return runEvalUsername; + return agentUsername; } /** - * Gets the agent user, username, from the shared test session. - * Throws an error if the session hasn't been created yet. + * DIAGNOSTIC (do-not-merge): dump the token-redacted lines of the sf CLI's own trace log that + * relate to the OAuth token exchange / login-rate response, so CI can show the HTTP status, + * endpoint, and any Retry-After for a `login rate exceeded` failure — settling whether the cap + * is per-org or per-IP. Uses SF_LOG_LEVEL=trace's log file (which @salesforce/core redacts of + * access/refresh tokens), NOT jsforce DEBUG (which would leak raw tokens into public CI logs). */ -export function getAgentUsername(): string | undefined { +export function dumpCliAuthTrace(label: string): void { if (!testSession) { - throw new Error('Test session not available. Call getTestSession() first.'); + console.log(`[AUTH-TRACE ${label}] no test session; nothing to dump`); + return; + } + // The testkit child shares HOME with the session; sf writes its log under /.sf. + const homes = Array.from(new Set([testSession.homeDir, testSession.dir].filter(Boolean))) as string[]; + const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; + let printedAny = false; + for (const home of homes) { + const sfDir = join(home, '.sf'); + if (!existsSync(sfDir)) { + continue; + } + for (const logFile of readdirSync(sfDir).filter((f) => f.endsWith('.log'))) { + const full = join(sfDir, logFile); + try { + const matches = readFileSync(full, 'utf8') + .split('\n') + .filter((l) => keyword.test(l)); + if (matches.length) { + printedAny = true; + console.log(`[AUTH-TRACE ${label}] ${full}: ${matches.length} matching line(s), last 40:`); + matches.slice(-40).forEach((l) => console.log(`[AUTH-TRACE] ${l}`)); + } + } catch { + // ignore unreadable/rotated log files + } + } + } + if (!printedAny) { + console.log( + `[AUTH-TRACE ${label}] no matching trace lines found under ${homes + .map((h) => join(h, '.sf')) + .join(', ')} (log path may differ; checked *.log)` + ); } - return agentUsername; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 9883810a8d327f2f08d3617c1e0bf1da28573b1e Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 21 Oct 2026 15:50:41 -0700 Subject: [PATCH 05/15] test(nuts): fix lint in auth-trace helper (unnecessary type assertion) [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Array.from(...) as string[] with a type-guard filter so @typescript-eslint/no-unnecessary-type-assertion passes. (Local hooks had missed it — stale node_modules was missing @eslint-react; reinstalled.) --- test/nuts/shared-setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index dfed61bf..49d6277a 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -305,7 +305,7 @@ export function dumpCliAuthTrace(label: string): void { return; } // The testkit child shares HOME with the session; sf writes its log under /.sf. - const homes = Array.from(new Set([testSession.homeDir, testSession.dir].filter(Boolean))) as string[]; + const homes = [...new Set([testSession.homeDir, testSession.dir].filter((d): d is string => Boolean(d)))]; const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; let printedAny = false; for (const home of homes) { From 860baa8d542a131624a416f5e7f4230e366e680f Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Thu, 22 Oct 2026 13:59:00 -0700 Subject: [PATCH 06/15] test(nuts): ride out login-rate with backoff on run-eval hooks [do-not-merge] Replace the do-not-merge SF_LOG_LEVEL=trace diagnostic with a login-rate retry helper (execCmdWithLoginRateBackoff) and apply it to the run-eval JSON and YAML before-hooks. The shared scratch org's per-org/per-runner login-rate budget is spent by setup + other NUTs before run-eval runs; retrying with backoff (up to 4 attempts, ~6 min total wait) lets the window recover during the serial waits. Hook timeouts raised to fit the backoff plus the ~18-min eval run. --- test/nuts/agent.test.run-eval.nut.ts | 33 +++++------- test/nuts/shared-setup.ts | 77 ++++++++++++++-------------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 63080acf..a85168f7 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,11 +18,13 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername, dumpCliAuthTrace } from './shared-setup.js'; +import { getTestSession, getUsername, execCmdWithLoginRateBackoff } from './shared-setup.js'; describe('agent test run-eval', function () { - // Increase timeout for setup since shared setup includes long waits and deployments - this.timeout(30 * 60 * 1000); // 30 minutes + // Increase timeout for setup since shared setup includes long waits and deployments, plus the + // run-eval before-hook rides out the `login rate exceeded` window with backoff (up to ~6 min of + // waiting) before an ~18-min eval run. + this.timeout(60 * 60 * 1000); // 60 minutes const mockProjectDir = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template', 'specs'); const jsonPayloadPath = join(mockProjectDir, 'eval-payload.json'); @@ -36,23 +38,15 @@ describe('agent test run-eval', function () { let jsonResult: RunEvalResult; before(async function () { - this.timeout(30 * 60 * 1000); // 30 minutes for setup + this.timeout(60 * 60 * 1000); // 60 minutes: setup + login-rate backoff + the eval run await getTestSession(); const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // DIAGNOSTIC (do-not-merge): run under SF_LOG_LEVEL=trace so the CLI's own (token-redacted) - // log records the OAuth token exchange; dump the relevant lines regardless of pass/fail so CI - // shows the `login rate exceeded` response's HTTP status/endpoint/Retry-After (per-org vs per-IP). - try { - const jsonExec = execCmd(jsonCommand, { - ensureExitCode: 0, - env: { ...process.env, SF_LOG_LEVEL: 'trace' }, - }); - expect(jsonExec.jsonOutput?.result).to.be.ok; - jsonResult = jsonExec.jsonOutput!.result; - } finally { - dumpCliAuthTrace('run-eval-json-before'); - } + // Retry with backoff to ride out the transient `login rate exceeded` window that the shared + // org's auth budget trips when run-eval's first grant fires after suite setup. + const jsonExec = await execCmdWithLoginRateBackoff(jsonCommand); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; }); describe('run-eval with JSON file', () => { @@ -80,9 +74,10 @@ describe('agent test run-eval', function () { // for the same OAuth-rate-limit reason as the JSON-payload run above. let yamlResult: RunEvalResult; - before(() => { + before(async function () { + this.timeout(30 * 60 * 1000); // 30 minutes: login-rate backoff + the eval run const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const output = execCmd(command, { ensureExitCode: 0 }); + const output = await execCmdWithLoginRateBackoff(command); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 49d6277a..362e835e 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,8 +15,7 @@ */ import { join } from 'node:path'; -import { existsSync, readdirSync, readFileSync } from 'node:fs'; -import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { Duration, TestSession, execCmd, type ExecCmdResult } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; @@ -292,50 +291,50 @@ export function getAgentUsername(): string | undefined { return agentUsername; } +/** Matches the transient Salesforce OAuth "login rate exceeded" error returned during a token grant. */ +const LOGIN_RATE_RE = /login rate exceeded/i; + /** - * DIAGNOSTIC (do-not-merge): dump the token-redacted lines of the sf CLI's own trace log that - * relate to the OAuth token exchange / login-rate response, so CI can show the HTTP status, - * endpoint, and any Retry-After for a `login rate exceeded` failure — settling whether the cap - * is per-org or per-IP. Uses SF_LOG_LEVEL=trace's log file (which @salesforce/core redacts of - * access/refresh tokens), NOT jsforce DEBUG (which would leak raw tokens into public CI logs). + * Execute a command that is prone to the transient `login rate exceeded` OAuth error, retrying + * with backoff to ride out the (short) rate-limit window. Each attempt runs a fresh `execCmd`; + * on a login-rate failure we wait and retry, on a clean exit we return the result, and on any + * other non-zero exit we fail fast with the command output. Throws once `maxAttempts` login-rate + * failures are exhausted. + * + * This exists because the single shared scratch org's per-org/per-runner login-rate budget is + * spent by suite setup + the other NUTs before the run-eval describe runs; retrying lets the + * window recover (nothing else runs during these serial waits) instead of failing the describe. */ -export function dumpCliAuthTrace(label: string): void { - if (!testSession) { - console.log(`[AUTH-TRACE ${label}] no test session; nothing to dump`); - return; - } - // The testkit child shares HOME with the session; sf writes its log under /.sf. - const homes = [...new Set([testSession.homeDir, testSession.dir].filter((d): d is string => Boolean(d)))]; - const keyword = /(oauth2\/token|login rate|retry-after|invalid_grant|statusCode|refresh token|services\/oauth2)/i; - let printedAny = false; - for (const home of homes) { - const sfDir = join(home, '.sf'); - if (!existsSync(sfDir)) { - continue; - } - for (const logFile of readdirSync(sfDir).filter((f) => f.endsWith('.log'))) { - const full = join(sfDir, logFile); - try { - const matches = readFileSync(full, 'utf8') - .split('\n') - .filter((l) => keyword.test(l)); - if (matches.length) { - printedAny = true; - console.log(`[AUTH-TRACE ${label}] ${full}: ${matches.length} matching line(s), last 40:`); - matches.slice(-40).forEach((l) => console.log(`[AUTH-TRACE] ${l}`)); - } - } catch { - // ignore unreadable/rotated log files +export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { + // Waits between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. + const delays = [Duration.seconds(60), Duration.seconds(120), Duration.seconds(180)]; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const exec = execCmd(command); + if (exec.shellOutput.code === 0) { + if (attempt > 1) { + console.log(`[LOGIN-RATE-BACKOFF] succeeded on attempt ${attempt}/${maxAttempts}: ${command}`); } + return exec; } - } - if (!printedAny) { + const output = `${exec.shellOutput.stdout ?? ''}\n${exec.shellOutput.stderr ?? ''}\n${JSON.stringify( + exec.jsonOutput ?? {} + )}`; + const isLoginRate = LOGIN_RATE_RE.test(output); + if (!isLoginRate || attempt === maxAttempts) { + throw new Error( + `Command failed (exit ${exec.shellOutput.code}) after ${attempt} attempt(s)${ + isLoginRate ? ' — login rate exceeded never cleared' : '' + }: ${command}\n${output}` + ); + } + const delay = delays[Math.min(attempt - 1, delays.length - 1)]; console.log( - `[AUTH-TRACE ${label}] no matching trace lines found under ${homes - .map((h) => join(h, '.sf')) - .join(', ')} (log path may differ; checked *.log)` + `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${delay.seconds}s before retry: ${command}` ); + await sleep(delay); // eslint-disable-line no-await-in-loop } + // Unreachable: the loop either returns on success or throws on the final attempt. + throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 283d6fb3aef9501453e5165a145c22d722d0a0d4 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 10:08:03 -0700 Subject: [PATCH 07/15] test(nuts): fix login-rate backoff never waiting (Duration identity) [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backoff passed a testkit-re-exported Duration to @salesforce/kit's sleep(), whose `instanceof Duration` check missed (different class copy), producing a NaN timeout that resolved instantly — so all 4 attempts fired within ~6s and the window never had time to clear. Pass plain millisecond numbers to sleep() instead. --- test/nuts/shared-setup.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 362e835e..9c6f6a1a 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -306,8 +306,11 @@ const LOGIN_RATE_RE = /login rate exceeded/i; * window recover (nothing else runs during these serial waits) instead of failing the describe. */ export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { - // Waits between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. - const delays = [Duration.seconds(60), Duration.seconds(120), Duration.seconds(180)]; + // Wait (in ms) between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. + // NOTE: pass a plain number to `sleep`, NOT a `Duration` — the `Duration` re-exported by testkit + // is a different class instance than @salesforce/kit's, so `sleep`'s `instanceof Duration` check + // misses, yielding a NaN timeout that resolves instantly (the bug that made the first run not wait). + const delaysMs = [60_000, 120_000, 180_000]; for (let attempt = 1; attempt <= maxAttempts; attempt++) { const exec = execCmd(command); if (exec.shellOutput.code === 0) { @@ -327,11 +330,13 @@ export async function execCmdWithLoginRateBackoff(command: string, maxAttempt }: ${command}\n${output}` ); } - const delay = delays[Math.min(attempt - 1, delays.length - 1)]; + const delayMs = delaysMs[Math.min(attempt - 1, delaysMs.length - 1)]; console.log( - `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${delay.seconds}s before retry: ${command}` + `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${ + delayMs / 1000 + }s before retry: ${command}` ); - await sleep(delay); // eslint-disable-line no-await-in-loop + await sleep(delayMs); // eslint-disable-line no-await-in-loop } // Unreachable: the loop either returns on success or throws on the final attempt. throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); From 2bc5f4ce5e1b7d04eff95a2e0af340dd55cde301 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 13:25:15 -0700 Subject: [PATCH 08/15] test(nuts): give run-eval its own scratch org (per-org isolation probe) [do-not-merge] Revert the login-rate backoff (CI run 39992637604 proved it: the Duration identity fix made sleep actually wait, but 6 min of backoff never cleared the window, and it pushed the suite into the 60-min job timeout). Instead, isolate run-eval on a DEDICATED TestSession/scratch org: - extract org provisioning into reusable provisionEvalOrg(session) - add getRunEvalTestSession()/getRunEvalUsername(); restore AGENT_USER_USERNAME to the shared org's agent user afterward so z2 publish isn't contaminated - rewire agent.test.run-eval.nut.ts to the dedicated org Discriminates per-ORG vs per-IP: if run-eval's first grant succeeds on a fresh org, the cap is per-org; if it still trips login rate exceeded, it's per-IP. --- test/nuts/agent.test.run-eval.nut.ts | 50 ++- test/nuts/shared-setup.ts | 437 +++++++++++++++------------ 2 files changed, 267 insertions(+), 220 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index a85168f7..e0a3e63c 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,12 +18,12 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getTestSession, getUsername, execCmdWithLoginRateBackoff } from './shared-setup.js'; +import { getRunEvalTestSession, getRunEvalUsername } from './shared-setup.js'; describe('agent test run-eval', function () { - // Increase timeout for setup since shared setup includes long waits and deployments, plus the - // run-eval before-hook rides out the `login rate exceeded` window with backoff (up to ~6 min of - // waiting) before an ~18-min eval run. + // run-eval gets its OWN dedicated scratch org (see shared-setup.ts, per-org isolation + // experiment), so this before-hook provisions a fresh org (~20-25 min: create + 2 deploys + + // Einstein wait) and then runs the ~18-min eval. Generous suite timeout accordingly. this.timeout(60 * 60 * 1000); // 60 minutes const mockProjectDir = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template', 'specs'); @@ -31,20 +31,18 @@ describe('agent test run-eval', function () { const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); - // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's - // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the - // parsed result across every test that asserts against it, so a rate-limit failure surfaces - // once rather than per-test. + // The JSON-payload run-eval is expensive (~18 min) and each invocation is a fresh OAuth token + // grant, so repeated invocations press the org's login-rate budget. Run it exactly once here + // and share the parsed result across every test that asserts against it, so a rate-limit + // failure surfaces once rather than per-test. let jsonResult: RunEvalResult; before(async function () { - this.timeout(60 * 60 * 1000); // 60 minutes: setup + login-rate backoff + the eval run - await getTestSession(); + this.timeout(60 * 60 * 1000); // 60 minutes: dedicated-org provisioning + the eval run + await getRunEvalTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Retry with backoff to ride out the transient `login rate exceeded` window that the shared - // org's auth budget trips when run-eval's first grant fires after suite setup. - const jsonExec = await execCmdWithLoginRateBackoff(jsonCommand); + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); expect(jsonExec.jsonOutput?.result).to.be.ok; jsonResult = jsonExec.jsonOutput!.result; }); @@ -75,9 +73,9 @@ describe('agent test run-eval', function () { let yamlResult: RunEvalResult; before(async function () { - this.timeout(30 * 60 * 1000); // 30 minutes: login-rate backoff + the eval run - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const output = await execCmdWithLoginRateBackoff(command); + this.timeout(30 * 60 * 1000); // 30 minutes: the eval run (org already provisioned) + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; + const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; }); @@ -96,7 +94,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -109,7 +107,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -117,7 +115,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -126,20 +124,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -162,7 +160,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -174,7 +172,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 9c6f6a1a..e14869a6 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -15,14 +15,14 @@ */ import { join } from 'node:path'; -import { Duration, TestSession, execCmd, type ExecCmdResult } from '@salesforce/cli-plugins-testkit'; +import { Duration, TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { ComponentSetBuilder, RequestStatus, type ScopedPostDeploy } from '@salesforce/source-deploy-retrieve'; import { Org, SfError, User, Lifecycle, Connection } from '@salesforce/core'; import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ -// Module-level variables to ensure only one TestSession is created +// The shared TestSession + one scratch org used by the bulk of the suite. let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; @@ -30,6 +30,20 @@ let agentUsername: string | undefined; let orgId: string | undefined; let orgPod: string | undefined; +// A DEDICATED TestSession + its own scratch org for the `agent test run-eval` NUTs. +// PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant trips `login rate +// exceeded` because the shared org's (or the CI runner IP's) login-rate budget is already +// spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME org did +// not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether the +// cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs +// share the runner IP → still fails). +let runEvalSession: TestSession | undefined; +let runEvalSessionPromise: Promise | undefined; + +// The deploy progress listeners attach to the global Lifecycle singleton, so register them +// exactly once even though two sessions get provisioned. +let deployListenersRegistered = false; + // Format a UTC timestamp as MM/DD/YYYY:HH:mm:ss for CI log correlation (Splunk-friendly). function fmtUtc(d: Date): string { const p = (n: number): string => String(n).padStart(2, '0'); @@ -65,6 +79,169 @@ async function waitForEinsteinReady(connection: Connection, maxAttempts = 30): P throw new Error(`Einstein AI did not become ready within ${timeoutSeconds} seconds timeout`); } +// Register the deploy progress logging once against the global Lifecycle singleton. The +// listeners reset their own progress counters on each `scopedPreDeploy`, so a single +// registration correctly handles the deploys of both the shared and the run-eval sessions. +function registerDeployListeners(): void { + if (deployListenersRegistered) { + return; + } + deployListenersRegistered = true; + + const lifecycle = Lifecycle.getInstance(); + let lastDeployed = 0; + let lastTotal = 0; + + lifecycle.on('scopedPreDeploy', () => { + console.log('[DEPLOY] Starting deployment...'); + // Reset progress tracking for new deployment + lastDeployed = 0; + lastTotal = 0; + return Promise.resolve(); + }); + + lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { + const deployResult = result.deployResult.response; + const status = deployResult.status; + const numberComponentErrors = deployResult.numberComponentErrors ?? 0; + const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; + const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; + const numberTestErrors = deployResult.numberTestErrors ?? 0; + const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; + const numberTestsTotal = deployResult.numberTestsTotal ?? 0; + + // Log progress during polling (only if changed) + if ( + (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && + numberComponentsTotal > 0 + ) { + console.log( + `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ + numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' + }` + ); + lastDeployed = numberComponentsDeployed; + lastTotal = numberComponentsTotal; + } + + // Log final status when deployment is complete + const isComplete = + status === RequestStatus.Succeeded || status === RequestStatus.Failed || status === RequestStatus.Canceled; + if (isComplete) { + console.log(`[DEPLOY] Deployment completed - Status: ${status}`); + console.log( + `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` + ); + if (numberTestsTotal > 0) { + console.log( + `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` + ); + } + const componentFailures = ensureArray(deployResult.details?.componentFailures); + if (componentFailures.length > 0) { + console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); + componentFailures.slice(0, 5).forEach((failure, idx) => { + console.log( + `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${failure.problemType ?? 'unknown'}: ${ + failure.problem ?? 'unknown' + }` + ); + }); + } + } + return Promise.resolve(); + }); +} + +/** + * Provision a freshly-created scratch org so it can run agent evaluations: assign the Einstein + * perm set to the admin, create the agent runtime user, wait for Einstein AI to come up, and + * deploy the agent metadata (first without, then with the AiEvaluationDefinition). Used for BOTH + * the shared session and the dedicated run-eval session. + * + * Side effect: sets `process.env.AGENT_USER_USERNAME` to THIS org's agent user, because the + * metadata deploys use it as an SDR `replaceWithEnv` token (see mock project sfdx-project.json). + * Any caller that provisions a second org must restore the env var afterward if a later deploy + * targets a different org. + */ +async function provisionEvalOrg( + session: TestSession +): Promise<{ username: string; agentUser: string; orgId: string; pod: string }> { + const defaultOrg = session.orgs.get('default'); + if (!defaultOrg?.username) { + throw new Error('No default scratch org found in the test session.'); + } + const username = defaultOrg.username; + console.log(`Using scratch org: ${username}`); + + const org = await Org.create({ aliasOrUsername: username }); + const connection = org.getConnection(); + + // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). + const resolvedOrgId = org.getOrgId(); + const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>('SELECT InstanceName FROM Organization'); + const pod = orgInfo.InstanceName; + console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${resolvedOrgId} pod=${pod}`); + + // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user + const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( + `SELECT Id, Name FROM User WHERE Username='${username}'` + ); + const user = await User.create({ org }); + await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); + console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); + + // Create agent user using the new CLI command + console.log('Creating agent user...'); + const agentUserResult = execCmd<{ username: string }>(`org create agent-user --target-org ${username} --json`, { + ensureExitCode: 0, + cli: 'sf', + }); + const agentUser = agentUserResult.jsonOutput?.result?.username; + if (!agentUser) { + throw new Error('Failed to create agent user: no username returned'); + } + console.log(`Agent user created: ${agentUser}`); + + // Wait for Einstein AI services to be ready + console.log('Waiting for Einstein AI services to be ready...'); + await waitForEinsteinReady(connection); + console.log('Einstein AI services are ready'); + + // Set environment variable for string replacement (SDR `replaceWithEnv`) used by the deploys. + process.env.AGENT_USER_USERNAME = agentUser; + process.env.SF_AAB_COMPILATION = 'false'; + + registerDeployListeners(); + + console.log('deploying metadata (no AiEvaluationDefinition)'); + const cs1 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(session.project.dir, 'noTest.xml'), + directoryPaths: [session.homeDir], + }, + }); + const deploy1 = await cs1.deploy({ usernameOrConnection: username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] First deployment completed successfully'); + + console.log('deploying metadata (AiEvaluationDefinition)'); + const cs2 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(session.project.dir, 'test.xml'), + directoryPaths: [session.homeDir], + }, + }); + const deploy2 = await cs2.deploy({ usernameOrConnection: username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] Second deployment completed successfully'); + console.log('[DEPLOY] All deployments complete, tests can now run'); + + return { username, agentUser, orgId: resolvedOrgId, pod }; +} + /** * Gets the shared TestSession with a scratch org. This ensures only one TestSession * is created per test run, even if called from multiple test files simultaneously. @@ -103,154 +280,10 @@ export async function getTestSession(): Promise { testSession = session; console.log('TestSession created successfully'); - // Get the scratch org username and assign permission set - const orgs = session.orgs; - const defaultOrg = orgs.get('default'); - - if (orgs && orgs.size > 0) { - if (defaultOrg?.username) { - console.log(`Using scratch org: ${defaultOrg.username}`); - const org = await Org.create({ aliasOrUsername: defaultOrg.username }); - const connection = org.getConnection(); - - // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). - orgId = org.getOrgId(); - const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>( - 'SELECT InstanceName FROM Organization' - ); - orgPod = orgInfo.InstanceName; - console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${orgId} pod=${orgPod}`); - - // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user - const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( - `SELECT Id, Name FROM User WHERE Username='${defaultOrg.username}'` - ); - const user = await User.create({ org }); - await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); - console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); - - // Create agent user using the new CLI command - console.log('Creating agent user...'); - const agentUserResult = execCmd<{ username: string }>( - `org create agent-user --target-org ${defaultOrg.username} --json`, - { ensureExitCode: 0, cli: 'sf' } - ); - - if (!agentUserResult.jsonOutput?.result?.username) { - throw new Error('Failed to create agent user: no username returned'); - } - - agentUsername = agentUserResult.jsonOutput.result.username; - console.log(`Agent user created: ${agentUsername}`); - - // Wait for Einstein AI services to be ready - console.log('Waiting for Einstein AI services to be ready...'); - await waitForEinsteinReady(connection); - console.log('Einstein AI services are ready'); - - // Set environment variable for string replacement - process.env.AGENT_USER_USERNAME = agentUsername; - process.env.SF_AAB_COMPILATION = 'false'; - - // Set up deploy event listeners to log progress - const lifecycle = Lifecycle.getInstance(); - let lastDeployed = 0; - let lastTotal = 0; - - lifecycle.on('scopedPreDeploy', () => { - console.log('[DEPLOY] Starting deployment...'); - // Reset progress tracking for new deployment - lastDeployed = 0; - lastTotal = 0; - return Promise.resolve(); - }); - - lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { - const deployResult = result.deployResult.response; - const status = deployResult.status; - const numberComponentErrors = deployResult.numberComponentErrors ?? 0; - const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; - const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; - const numberTestErrors = deployResult.numberTestErrors ?? 0; - const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; - const numberTestsTotal = deployResult.numberTestsTotal ?? 0; - - // Log progress during polling (only if changed) - if ( - (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && - numberComponentsTotal > 0 - ) { - console.log( - `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ - numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' - }` - ); - lastDeployed = numberComponentsDeployed; - lastTotal = numberComponentsTotal; - } - - // Log final status when deployment is complete - const isComplete = - status === RequestStatus.Succeeded || - status === RequestStatus.Failed || - status === RequestStatus.Canceled; - if (isComplete) { - console.log(`[DEPLOY] Deployment completed - Status: ${status}`); - console.log( - `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` - ); - if (numberTestsTotal > 0) { - console.log( - `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` - ); - } - const componentFailures = ensureArray(deployResult.details?.componentFailures); - if (componentFailures.length > 0) { - console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); - componentFailures.slice(0, 5).forEach((failure, idx) => { - console.log( - `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${ - failure.problemType ?? 'unknown' - }: ${failure.problem ?? 'unknown'}` - ); - }); - } - } - return Promise.resolve(); - }); - - console.log('deploying metadata (no AiEvaluationDefinition)'); - - const cs1 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(testSession.project.dir, 'noTest.xml'), - directoryPaths: [testSession.homeDir], - }, - }); - const deploy1 = await cs1.deploy({ usernameOrConnection: defaultOrg.username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] First deployment completed successfully'); - - // Reset for second deployment - lastDeployed = 0; - lastTotal = 0; - - console.log('deploying metadata (AiEvaluationDefinition)'); - - const cs2 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(testSession.project.dir, 'test.xml'), - directoryPaths: [testSession.homeDir], - }, - }); - const deploy2 = await cs2.deploy({ usernameOrConnection: defaultOrg.username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] Second deployment completed successfully'); - console.log('[DEPLOY] All deployments complete, tests can now run'); - } - } + const provisioned = await provisionEvalOrg(session); + agentUsername = provisioned.agentUser; + orgId = provisioned.orgId; + orgPod = provisioned.pod; return session; } catch (e) { @@ -291,55 +324,71 @@ export function getAgentUsername(): string | undefined { return agentUsername; } -/** Matches the transient Salesforce OAuth "login rate exceeded" error returned during a token grant. */ -const LOGIN_RATE_RE = /login rate exceeded/i; - /** - * Execute a command that is prone to the transient `login rate exceeded` OAuth error, retrying - * with backoff to ride out the (short) rate-limit window. Each attempt runs a fresh `execCmd`; - * on a login-rate failure we wait and retry, on a clean exit we return the result, and on any - * other non-zero exit we fail fast with the command output. Throws once `maxAttempts` login-rate - * failures are exhausted. - * - * This exists because the single shared scratch org's per-org/per-runner login-rate budget is - * spent by suite setup + the other NUTs before the run-eval describe runs; retrying lets the - * window recover (nothing else runs during these serial waits) instead of failing the describe. + * Gets the DEDICATED run-eval TestSession, creating its own separate scratch org (isolated from + * the shared org used by the rest of the suite). See the per-org isolation experiment note above. + * After provisioning, restores `process.env.AGENT_USER_USERNAME` to the shared org's agent user so + * later shared-org deploys (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated + * with this org's agent user. */ -export async function execCmdWithLoginRateBackoff(command: string, maxAttempts = 4): Promise> { - // Wait (in ms) between attempts (attempt 1->2, 2->3, 3->4); worst case ~6 min of waiting total. - // NOTE: pass a plain number to `sleep`, NOT a `Duration` — the `Duration` re-exported by testkit - // is a different class instance than @salesforce/kit's, so `sleep`'s `instanceof Duration` check - // misses, yielding a NaN timeout that resolves instantly (the bug that made the first run not wait). - const delaysMs = [60_000, 120_000, 180_000]; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const exec = execCmd(command); - if (exec.shellOutput.code === 0) { - if (attempt > 1) { - console.log(`[LOGIN-RATE-BACKOFF] succeeded on attempt ${attempt}/${maxAttempts}: ${command}`); +export async function getRunEvalTestSession(): Promise { + if (runEvalSession) { + return runEvalSession; + } + if (runEvalSessionPromise) { + return runEvalSessionPromise; + } + + // @ts-expect-error it can throw an error + runEvalSessionPromise = (async (): Promise => { + try { + console.log('Creating DEDICATED run-eval TestSession with its own scratch org...'); + const session = await TestSession.create({ + project: { + sourceDir: join('test', 'mock-projects', 'agent-generate-template'), + }, + devhubAuthStrategy: 'AUTO', + scratchOrgs: [ + { + alias: 'default', + setDefault: true, + config: 'config/project-scratch-def.json', + }, + ], + }); + + runEvalSession = session; + console.log('run-eval TestSession created successfully'); + + const provisioned = await provisionEvalOrg(session); + console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); + + // Restore the SDR replacement env var to the shared org's agent user, so later shared-org + // deploys are not contaminated by the run-eval org's agent user. + if (agentUsername) { + process.env.AGENT_USER_USERNAME = agentUsername; } - return exec; - } - const output = `${exec.shellOutput.stdout ?? ''}\n${exec.shellOutput.stderr ?? ''}\n${JSON.stringify( - exec.jsonOutput ?? {} - )}`; - const isLoginRate = LOGIN_RATE_RE.test(output); - if (!isLoginRate || attempt === maxAttempts) { - throw new Error( - `Command failed (exit ${exec.shellOutput.code}) after ${attempt} attempt(s)${ - isLoginRate ? ' — login rate exceeded never cleared' : '' - }: ${command}\n${output}` - ); + + return session; + } catch (e) { + console.log('XXXXXX RUN-EVAL SESSION ERROR XXXXXXX'); + console.dir(SfError.wrap(e).toObject()); + void Promise.reject(SfError.wrap(e)); } - const delayMs = delaysMs[Math.min(attempt - 1, delaysMs.length - 1)]; - console.log( - `[LOGIN-RATE-BACKOFF] login rate exceeded on attempt ${attempt}/${maxAttempts}; waiting ${ - delayMs / 1000 - }s before retry: ${command}` - ); - await sleep(delayMs); // eslint-disable-line no-await-in-loop + })(); + + return runEvalSessionPromise; +} + +/** + * Gets the scratch org username from the dedicated run-eval test session. + * Throws an error if the session hasn't been created yet. + */ +export function getRunEvalUsername(): string { + if (!runEvalSession) { + throw new Error('Run-eval test session not available. Call getRunEvalTestSession() first.'); } - // Unreachable: the loop either returns on success or throws on the final attempt. - throw new Error(`Command failed after ${maxAttempts} attempts: ${command}`); + return runEvalSession.orgs.get('default')!.username!; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 2d7d42146ab86f821a8dee220d64798e618d14a4 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Fri, 23 Oct 2026 15:59:00 -0700 Subject: [PATCH 09/15] test(nuts): run-eval isolation via 2nd org in one session, not 2nd session [do-not-merge] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior attempt (2bc5f4c) failed fast: a second TestSession.create() throws 'Attempted to wrap cwd which is already wrapped' and corrupts the first session (testkit wraps process.cwd() globally — one TestSession per process). Keep the per-org isolation experiment but change the mechanism: declare a second scratch org (alias 'runeval') in the SAME shared TestSession, provision it lazily, and point the run-eval NUTs at it. provisionEvalOrg is now parameterized by alias. Still discriminates per-ORG vs per-IP for the login-rate cap. --- test/nuts/shared-setup.ts | 121 +++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 68 deletions(-) diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index e14869a6..586fb8be 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -30,18 +30,23 @@ let agentUsername: string | undefined; let orgId: string | undefined; let orgPod: string | undefined; -// A DEDICATED TestSession + its own scratch org for the `agent test run-eval` NUTs. -// PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant trips `login rate -// exceeded` because the shared org's (or the CI runner IP's) login-rate budget is already -// spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME org did -// not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether the -// cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs +// A DEDICATED SECOND scratch org (alias `runeval`) in the SAME TestSession for the +// `agent test run-eval` NUTs. PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant +// trips `login rate exceeded` because the shared org's (or the CI runner IP's) login-rate budget +// is already spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME +// org did not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether +// the cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs // share the runner IP → still fails). -let runEvalSession: TestSession | undefined; -let runEvalSessionPromise: Promise | undefined; +// +// NOTE: this must be a second ORG in the one TestSession, NOT a second TestSession — testkit wraps +// process.cwd() globally and a second TestSession.create() throws "Attempted to wrap cwd which is +// already wrapped" and corrupts the first session. +const RUN_EVAL_ORG_ALIAS = 'runeval'; +let runEvalUsername: string | undefined; +let runEvalProvisionPromise: Promise | undefined; // The deploy progress listeners attach to the global Lifecycle singleton, so register them -// exactly once even though two sessions get provisioned. +// exactly once even though two orgs get provisioned. let deployListenersRegistered = false; // Format a UTC timestamp as MM/DD/YYYY:HH:mm:ss for CI log correlation (Splunk-friendly). @@ -165,14 +170,15 @@ function registerDeployListeners(): void { * targets a different org. */ async function provisionEvalOrg( - session: TestSession + session: TestSession, + alias = 'default' ): Promise<{ username: string; agentUser: string; orgId: string; pod: string }> { - const defaultOrg = session.orgs.get('default'); - if (!defaultOrg?.username) { - throw new Error('No default scratch org found in the test session.'); + const scratchOrg = session.orgs.get(alias); + if (!scratchOrg?.username) { + throw new Error(`No '${alias}' scratch org found in the test session.`); } - const username = defaultOrg.username; - console.log(`Using scratch org: ${username}`); + const username = scratchOrg.username; + console.log(`Using scratch org (${alias}): ${username}`); const org = await Org.create({ aliasOrUsername: username }); const connection = org.getConnection(); @@ -274,6 +280,13 @@ export async function getTestSession(): Promise { setDefault: true, config: 'config/project-scratch-def.json', }, + { + // Dedicated org for the run-eval NUTs (per-org login-rate isolation experiment). + // Created here, provisioned lazily in getRunEvalTestSession(). + alias: RUN_EVAL_ORG_ALIAS, + setDefault: false, + config: 'config/project-scratch-def.json', + }, ], }); @@ -325,70 +338,42 @@ export function getAgentUsername(): string | undefined { } /** - * Gets the DEDICATED run-eval TestSession, creating its own separate scratch org (isolated from - * the shared org used by the rest of the suite). See the per-org isolation experiment note above. - * After provisioning, restores `process.env.AGENT_USER_USERNAME` to the shared org's agent user so - * later shared-org deploys (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated - * with this org's agent user. + * Ensures the DEDICATED run-eval scratch org (alias `runeval`, created alongside the shared org in + * the one TestSession) is provisioned and ready. Provisioned lazily and exactly once. See the + * per-org isolation experiment note above. After provisioning, restores + * `process.env.AGENT_USER_USERNAME` to the shared org's agent user so later shared-org deploys + * (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated with this org's agent user. */ export async function getRunEvalTestSession(): Promise { - if (runEvalSession) { - return runEvalSession; - } - if (runEvalSessionPromise) { - return runEvalSessionPromise; - } - - // @ts-expect-error it can throw an error - runEvalSessionPromise = (async (): Promise => { - try { - console.log('Creating DEDICATED run-eval TestSession with its own scratch org...'); - const session = await TestSession.create({ - project: { - sourceDir: join('test', 'mock-projects', 'agent-generate-template'), - }, - devhubAuthStrategy: 'AUTO', - scratchOrgs: [ - { - alias: 'default', - setDefault: true, - config: 'config/project-scratch-def.json', - }, - ], - }); - - runEvalSession = session; - console.log('run-eval TestSession created successfully'); - - const provisioned = await provisionEvalOrg(session); - console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); - - // Restore the SDR replacement env var to the shared org's agent user, so later shared-org - // deploys are not contaminated by the run-eval org's agent user. - if (agentUsername) { - process.env.AGENT_USER_USERNAME = agentUsername; - } - - return session; - } catch (e) { - console.log('XXXXXX RUN-EVAL SESSION ERROR XXXXXXX'); - console.dir(SfError.wrap(e).toObject()); - void Promise.reject(SfError.wrap(e)); + // Ensure the shared session (which also created the `runeval` org) exists first. + const session = await getTestSession(); + + runEvalProvisionPromise ??= (async (): Promise => { + console.log('Provisioning DEDICATED run-eval scratch org...'); + const provisioned = await provisionEvalOrg(session, RUN_EVAL_ORG_ALIAS); + runEvalUsername = provisioned.username; + console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); + + // Restore the SDR replacement env var to the shared org's agent user, so later shared-org + // deploys are not contaminated by the run-eval org's agent user. + if (agentUsername) { + process.env.AGENT_USER_USERNAME = agentUsername; } })(); + await runEvalProvisionPromise; - return runEvalSessionPromise; + return session; } /** - * Gets the scratch org username from the dedicated run-eval test session. - * Throws an error if the session hasn't been created yet. + * Gets the username of the dedicated run-eval scratch org. + * Throws an error if it hasn't been provisioned yet. */ export function getRunEvalUsername(): string { - if (!runEvalSession) { - throw new Error('Run-eval test session not available. Call getRunEvalTestSession() first.'); + if (!runEvalUsername) { + throw new Error('Run-eval org not provisioned. Call getRunEvalTestSession() first.'); } - return runEvalSession.orgs.get('default')!.username!; + return runEvalUsername; } // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + From 5a32b58e5047be2eb81c3e67e4d44781a4515f71 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Mon, 26 Oct 2026 11:41:13 -0700 Subject: [PATCH 10/15] test(nuts): revert run-eval per-org isolation; cap is per-IP [do-not-merge] CI run 40037029393 proved the dedicated-org experiment does not help: run-eval tripped 'login rate exceeded' on its first token grant against a fresh scratch org on a different pod/orgId than the shared org. A brand-new org with its own login budget failing means the cap is not per-org. The only constant across the shared org, the dedicated org, and all the different pods is the CI runner IP -> the budget is spent IP-wide by the rest of the suite before run-eval runs. Both isolation levers are now refuted in CI (dedicated user c2bb5f9, dedicated org 2bc5f4c/2d7d421). Restore shared-setup.ts and run-eval.nut.ts to the 8b6f429 shared-session baseline (de-dup + NUT-CONTEXT/NUT-FAILURE correlation logging kept). --- test/nuts/agent.test.run-eval.nut.ts | 45 ++-- test/nuts/shared-setup.ts | 383 +++++++++++---------------- 2 files changed, 170 insertions(+), 258 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index e0a3e63c..3533be5b 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -18,30 +18,28 @@ import { join } from 'node:path'; import { expect } from 'chai'; import { execCmd } from '@salesforce/cli-plugins-testkit'; import type { RunEvalResult } from '../../src/commands/agent/test/run-eval.js'; -import { getRunEvalTestSession, getRunEvalUsername } from './shared-setup.js'; +import { getTestSession, getUsername } from './shared-setup.js'; describe('agent test run-eval', function () { - // run-eval gets its OWN dedicated scratch org (see shared-setup.ts, per-org isolation - // experiment), so this before-hook provisions a fresh org (~20-25 min: create + 2 deploys + - // Einstein wait) and then runs the ~18-min eval. Generous suite timeout accordingly. - this.timeout(60 * 60 * 1000); // 60 minutes + // Increase timeout for setup since shared setup includes long waits and deployments + this.timeout(30 * 60 * 1000); // 30 minutes const mockProjectDir = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template', 'specs'); const jsonPayloadPath = join(mockProjectDir, 'eval-payload.json'); const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); - // The JSON-payload run-eval is expensive (~18 min) and each invocation is a fresh OAuth token - // grant, so repeated invocations press the org's login-rate budget. Run it exactly once here - // and share the parsed result across every test that asserts against it, so a rate-limit - // failure surfaces once rather than per-test. + // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's + // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the + // parsed result across every test that asserts against it, so a rate-limit failure surfaces + // once rather than per-test. let jsonResult: RunEvalResult; before(async function () { - this.timeout(60 * 60 * 1000); // 60 minutes: dedicated-org provisioning + the eval run - await getRunEvalTestSession(); + this.timeout(30 * 60 * 1000); // 30 minutes for setup + await getTestSession(); - const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); expect(jsonExec.jsonOutput?.result).to.be.ok; jsonResult = jsonExec.jsonOutput!.result; @@ -72,9 +70,8 @@ describe('agent test run-eval', function () { // for the same OAuth-rate-limit reason as the JSON-payload run above. let yamlResult: RunEvalResult; - before(async function () { - this.timeout(30 * 60 * 1000); // 30 minutes: the eval run (org already provisioned) - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getRunEvalUsername()} --json`; + before(() => { + const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; const output = execCmd(command, { ensureExitCode: 0 }); expect(output.jsonOutput?.result).to.be.ok; yamlResult = output.jsonOutput!.result; @@ -94,7 +91,7 @@ describe('agent test run-eval', function () { }); it('should handle YAML spec with contextVariables', async () => { - const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables @@ -107,7 +104,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; @@ -115,7 +112,7 @@ describe('agent test run-eval', function () { }); it('should use custom batch size', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; const result = execCmd(command, { ensureExitCode: 0 }); expect(result.jsonOutput?.result).to.be.ok; @@ -124,20 +121,20 @@ describe('agent test run-eval', function () { it('should support different result formats', async () => { // Test human format (default) - const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getRunEvalUsername()}`; + const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); expect(humanResult.shellOutput.stdout).to.be.ok; expect(humanResult.shellOutput.stdout).to.be.a('string'); // Test tap format - const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getRunEvalUsername()}`; + const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); expect(tapResult.shellOutput.stdout).to.include('TAP version'); // Test junit format - const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getRunEvalUsername()}`; + const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); expect(junitResult.shellOutput.stdout).to.include(' { it('should fail with invalid JSON payload', async () => { const invalidJson = join(mockProjectDir, 'invalid-payload.json'); - const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${invalidJson} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -160,7 +157,7 @@ describe('agent test run-eval', function () { }); it('should fail when agent not found', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name NonExistentAgent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); @@ -172,7 +169,7 @@ describe('agent test run-eval', function () { }); it('should require --spec flag', async () => { - const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getRunEvalUsername()} --json`; + const command = `agent test run-eval --api-name Local_Info_Agent --target-org ${getUsername()} --json`; try { execCmd(command, { ensureExitCode: 0 }); diff --git a/test/nuts/shared-setup.ts b/test/nuts/shared-setup.ts index 586fb8be..f3b9176c 100644 --- a/test/nuts/shared-setup.ts +++ b/test/nuts/shared-setup.ts @@ -22,7 +22,7 @@ import { sleep, ensureArray } from '@salesforce/kit'; /* eslint-disable no-console */ -// The shared TestSession + one scratch org used by the bulk of the suite. +// Module-level variables to ensure only one TestSession is created let testSession: TestSession | undefined; let testSessionPromise: Promise | undefined; let agentUsername: string | undefined; @@ -30,25 +30,6 @@ let agentUsername: string | undefined; let orgId: string | undefined; let orgPod: string | undefined; -// A DEDICATED SECOND scratch org (alias `runeval`) in the SAME TestSession for the -// `agent test run-eval` NUTs. PER-ORG ISOLATION EXPERIMENT: run-eval's first OAuth token grant -// trips `login rate exceeded` because the shared org's (or the CI runner IP's) login-rate budget -// is already spent by the rest of the suite before run-eval runs. A dedicated USER in the SAME -// org did not help (the cap is not per-user). Giving run-eval its OWN scratch org tests whether -// the cap is per-ORG (a fresh org has its own budget → run-eval succeeds) or per-IP (all orgs -// share the runner IP → still fails). -// -// NOTE: this must be a second ORG in the one TestSession, NOT a second TestSession — testkit wraps -// process.cwd() globally and a second TestSession.create() throws "Attempted to wrap cwd which is -// already wrapped" and corrupts the first session. -const RUN_EVAL_ORG_ALIAS = 'runeval'; -let runEvalUsername: string | undefined; -let runEvalProvisionPromise: Promise | undefined; - -// The deploy progress listeners attach to the global Lifecycle singleton, so register them -// exactly once even though two orgs get provisioned. -let deployListenersRegistered = false; - // Format a UTC timestamp as MM/DD/YYYY:HH:mm:ss for CI log correlation (Splunk-friendly). function fmtUtc(d: Date): string { const p = (n: number): string => String(n).padStart(2, '0'); @@ -84,170 +65,6 @@ async function waitForEinsteinReady(connection: Connection, maxAttempts = 30): P throw new Error(`Einstein AI did not become ready within ${timeoutSeconds} seconds timeout`); } -// Register the deploy progress logging once against the global Lifecycle singleton. The -// listeners reset their own progress counters on each `scopedPreDeploy`, so a single -// registration correctly handles the deploys of both the shared and the run-eval sessions. -function registerDeployListeners(): void { - if (deployListenersRegistered) { - return; - } - deployListenersRegistered = true; - - const lifecycle = Lifecycle.getInstance(); - let lastDeployed = 0; - let lastTotal = 0; - - lifecycle.on('scopedPreDeploy', () => { - console.log('[DEPLOY] Starting deployment...'); - // Reset progress tracking for new deployment - lastDeployed = 0; - lastTotal = 0; - return Promise.resolve(); - }); - - lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { - const deployResult = result.deployResult.response; - const status = deployResult.status; - const numberComponentErrors = deployResult.numberComponentErrors ?? 0; - const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; - const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; - const numberTestErrors = deployResult.numberTestErrors ?? 0; - const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; - const numberTestsTotal = deployResult.numberTestsTotal ?? 0; - - // Log progress during polling (only if changed) - if ( - (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && - numberComponentsTotal > 0 - ) { - console.log( - `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ - numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' - }` - ); - lastDeployed = numberComponentsDeployed; - lastTotal = numberComponentsTotal; - } - - // Log final status when deployment is complete - const isComplete = - status === RequestStatus.Succeeded || status === RequestStatus.Failed || status === RequestStatus.Canceled; - if (isComplete) { - console.log(`[DEPLOY] Deployment completed - Status: ${status}`); - console.log( - `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` - ); - if (numberTestsTotal > 0) { - console.log( - `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` - ); - } - const componentFailures = ensureArray(deployResult.details?.componentFailures); - if (componentFailures.length > 0) { - console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); - componentFailures.slice(0, 5).forEach((failure, idx) => { - console.log( - `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${failure.problemType ?? 'unknown'}: ${ - failure.problem ?? 'unknown' - }` - ); - }); - } - } - return Promise.resolve(); - }); -} - -/** - * Provision a freshly-created scratch org so it can run agent evaluations: assign the Einstein - * perm set to the admin, create the agent runtime user, wait for Einstein AI to come up, and - * deploy the agent metadata (first without, then with the AiEvaluationDefinition). Used for BOTH - * the shared session and the dedicated run-eval session. - * - * Side effect: sets `process.env.AGENT_USER_USERNAME` to THIS org's agent user, because the - * metadata deploys use it as an SDR `replaceWithEnv` token (see mock project sfdx-project.json). - * Any caller that provisions a second org must restore the env var afterward if a later deploy - * targets a different org. - */ -async function provisionEvalOrg( - session: TestSession, - alias = 'default' -): Promise<{ username: string; agentUser: string; orgId: string; pod: string }> { - const scratchOrg = session.orgs.get(alias); - if (!scratchOrg?.username) { - throw new Error(`No '${alias}' scratch org found in the test session.`); - } - const username = scratchOrg.username; - console.log(`Using scratch org (${alias}): ${username}`); - - const org = await Org.create({ aliasOrUsername: username }); - const connection = org.getConnection(); - - // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). - const resolvedOrgId = org.getOrgId(); - const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>('SELECT InstanceName FROM Organization'); - const pod = orgInfo.InstanceName; - console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${resolvedOrgId} pod=${pod}`); - - // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user - const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( - `SELECT Id, Name FROM User WHERE Username='${username}'` - ); - const user = await User.create({ org }); - await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); - console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); - - // Create agent user using the new CLI command - console.log('Creating agent user...'); - const agentUserResult = execCmd<{ username: string }>(`org create agent-user --target-org ${username} --json`, { - ensureExitCode: 0, - cli: 'sf', - }); - const agentUser = agentUserResult.jsonOutput?.result?.username; - if (!agentUser) { - throw new Error('Failed to create agent user: no username returned'); - } - console.log(`Agent user created: ${agentUser}`); - - // Wait for Einstein AI services to be ready - console.log('Waiting for Einstein AI services to be ready...'); - await waitForEinsteinReady(connection); - console.log('Einstein AI services are ready'); - - // Set environment variable for string replacement (SDR `replaceWithEnv`) used by the deploys. - process.env.AGENT_USER_USERNAME = agentUser; - process.env.SF_AAB_COMPILATION = 'false'; - - registerDeployListeners(); - - console.log('deploying metadata (no AiEvaluationDefinition)'); - const cs1 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(session.project.dir, 'noTest.xml'), - directoryPaths: [session.homeDir], - }, - }); - const deploy1 = await cs1.deploy({ usernameOrConnection: username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] First deployment completed successfully'); - - console.log('deploying metadata (AiEvaluationDefinition)'); - const cs2 = await ComponentSetBuilder.build({ - manifest: { - manifestPath: join(session.project.dir, 'test.xml'), - directoryPaths: [session.homeDir], - }, - }); - const deploy2 = await cs2.deploy({ usernameOrConnection: username }); - // pollStatus waits until deployment completes - will throw if deployment fails - await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); - console.log('[DEPLOY] Second deployment completed successfully'); - console.log('[DEPLOY] All deployments complete, tests can now run'); - - return { username, agentUser, orgId: resolvedOrgId, pod }; -} - /** * Gets the shared TestSession with a scratch org. This ensures only one TestSession * is created per test run, even if called from multiple test files simultaneously. @@ -280,23 +97,160 @@ export async function getTestSession(): Promise { setDefault: true, config: 'config/project-scratch-def.json', }, - { - // Dedicated org for the run-eval NUTs (per-org login-rate isolation experiment). - // Created here, provisioned lazily in getRunEvalTestSession(). - alias: RUN_EVAL_ORG_ALIAS, - setDefault: false, - config: 'config/project-scratch-def.json', - }, ], }); testSession = session; console.log('TestSession created successfully'); - const provisioned = await provisionEvalOrg(session); - agentUsername = provisioned.agentUser; - orgId = provisioned.orgId; - orgPod = provisioned.pod; + // Get the scratch org username and assign permission set + const orgs = session.orgs; + const defaultOrg = orgs.get('default'); + + if (orgs && orgs.size > 0) { + if (defaultOrg?.username) { + console.log(`Using scratch org: ${defaultOrg.username}`); + const org = await Org.create({ aliasOrUsername: defaultOrg.username }); + const connection = org.getConnection(); + + // Capture non-sensitive org identifiers for CI failure correlation (orgId + pod + UTC timestamp). + orgId = org.getOrgId(); + const orgInfo = await connection.singleRecordQuery<{ InstanceName: string }>( + 'SELECT InstanceName FROM Organization' + ); + orgPod = orgInfo.InstanceName; + console.log(`[NUT-CONTEXT] utc=${fmtUtc(new Date())} orgId=${orgId} pod=${orgPod}`); + + // assign the EinsteinGPTPromptTemplateManager to the scratch org admin user + const queryResult = await connection.singleRecordQuery<{ Id: string; Name: string }>( + `SELECT Id, Name FROM User WHERE Username='${defaultOrg.username}'` + ); + const user = await User.create({ org }); + await user.assignPermissionSets(queryResult.Id, ['EinsteinGPTPromptTemplateManager']); + console.log(`Permission set assigned to scratch org user: ${queryResult.Name}`); + + // Create agent user using the new CLI command + console.log('Creating agent user...'); + const agentUserResult = execCmd<{ username: string }>( + `org create agent-user --target-org ${defaultOrg.username} --json`, + { ensureExitCode: 0, cli: 'sf' } + ); + + if (!agentUserResult.jsonOutput?.result?.username) { + throw new Error('Failed to create agent user: no username returned'); + } + + agentUsername = agentUserResult.jsonOutput.result.username; + console.log(`Agent user created: ${agentUsername}`); + + // Wait for Einstein AI services to be ready + console.log('Waiting for Einstein AI services to be ready...'); + await waitForEinsteinReady(connection); + console.log('Einstein AI services are ready'); + + // Set environment variable for string replacement + process.env.AGENT_USER_USERNAME = agentUsername; + process.env.SF_AAB_COMPILATION = 'false'; + + // Set up deploy event listeners to log progress + const lifecycle = Lifecycle.getInstance(); + let lastDeployed = 0; + let lastTotal = 0; + + lifecycle.on('scopedPreDeploy', () => { + console.log('[DEPLOY] Starting deployment...'); + // Reset progress tracking for new deployment + lastDeployed = 0; + lastTotal = 0; + return Promise.resolve(); + }); + + lifecycle.on('scopedPostDeploy', (result: ScopedPostDeploy) => { + const deployResult = result.deployResult.response; + const status = deployResult.status; + const numberComponentErrors = deployResult.numberComponentErrors ?? 0; + const numberComponentsDeployed = deployResult.numberComponentsDeployed ?? 0; + const numberComponentsTotal = deployResult.numberComponentsTotal ?? 0; + const numberTestErrors = deployResult.numberTestErrors ?? 0; + const numberTestsCompleted = deployResult.numberTestsCompleted ?? 0; + const numberTestsTotal = deployResult.numberTestsTotal ?? 0; + + // Log progress during polling (only if changed) + if ( + (numberComponentsDeployed !== lastDeployed || numberComponentsTotal !== lastTotal) && + numberComponentsTotal > 0 + ) { + console.log( + `[DEPLOY] Progress: ${numberComponentsDeployed}/${numberComponentsTotal} components deployed${ + numberComponentErrors > 0 ? `, ${numberComponentErrors} errors` : '' + }` + ); + lastDeployed = numberComponentsDeployed; + lastTotal = numberComponentsTotal; + } + + // Log final status when deployment is complete + const isComplete = + status === RequestStatus.Succeeded || + status === RequestStatus.Failed || + status === RequestStatus.Canceled; + if (isComplete) { + console.log(`[DEPLOY] Deployment completed - Status: ${status}`); + console.log( + `[DEPLOY] Components: ${numberComponentsDeployed}/${numberComponentsTotal} deployed, ${numberComponentErrors} errors` + ); + if (numberTestsTotal > 0) { + console.log( + `[DEPLOY] Tests: ${numberTestsCompleted}/${numberTestsTotal} completed, ${numberTestErrors} errors` + ); + } + const componentFailures = ensureArray(deployResult.details?.componentFailures); + if (componentFailures.length > 0) { + console.log(`[DEPLOY] Component failures: ${componentFailures.length}`); + componentFailures.slice(0, 5).forEach((failure, idx) => { + console.log( + `[DEPLOY] Failure ${idx + 1}: ${failure.fullName ?? 'unknown'} - ${ + failure.problemType ?? 'unknown' + }: ${failure.problem ?? 'unknown'}` + ); + }); + } + } + return Promise.resolve(); + }); + + console.log('deploying metadata (no AiEvaluationDefinition)'); + + const cs1 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(testSession.project.dir, 'noTest.xml'), + directoryPaths: [testSession.homeDir], + }, + }); + const deploy1 = await cs1.deploy({ usernameOrConnection: defaultOrg.username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy1.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] First deployment completed successfully'); + + // Reset for second deployment + lastDeployed = 0; + lastTotal = 0; + + console.log('deploying metadata (AiEvaluationDefinition)'); + + const cs2 = await ComponentSetBuilder.build({ + manifest: { + manifestPath: join(testSession.project.dir, 'test.xml'), + directoryPaths: [testSession.homeDir], + }, + }); + const deploy2 = await cs2.deploy({ usernameOrConnection: defaultOrg.username }); + // pollStatus waits until deployment completes - will throw if deployment fails + await deploy2.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(30) }); + console.log('[DEPLOY] Second deployment completed successfully'); + console.log('[DEPLOY] All deployments complete, tests can now run'); + } + } return session; } catch (e) { @@ -337,45 +291,6 @@ export function getAgentUsername(): string | undefined { return agentUsername; } -/** - * Ensures the DEDICATED run-eval scratch org (alias `runeval`, created alongside the shared org in - * the one TestSession) is provisioned and ready. Provisioned lazily and exactly once. See the - * per-org isolation experiment note above. After provisioning, restores - * `process.env.AGENT_USER_USERNAME` to the shared org's agent user so later shared-org deploys - * (e.g. z2 publish of Willie_Resort_Manager.agent) are not contaminated with this org's agent user. - */ -export async function getRunEvalTestSession(): Promise { - // Ensure the shared session (which also created the `runeval` org) exists first. - const session = await getTestSession(); - - runEvalProvisionPromise ??= (async (): Promise => { - console.log('Provisioning DEDICATED run-eval scratch org...'); - const provisioned = await provisionEvalOrg(session, RUN_EVAL_ORG_ALIAS); - runEvalUsername = provisioned.username; - console.log(`[NUT-CONTEXT] run-eval dedicated org username=${provisioned.username} orgId=${provisioned.orgId}`); - - // Restore the SDR replacement env var to the shared org's agent user, so later shared-org - // deploys are not contaminated by the run-eval org's agent user. - if (agentUsername) { - process.env.AGENT_USER_USERNAME = agentUsername; - } - })(); - await runEvalProvisionPromise; - - return session; -} - -/** - * Gets the username of the dedicated run-eval scratch org. - * Throws an error if it hasn't been provisioned yet. - */ -export function getRunEvalUsername(): string { - if (!runEvalUsername) { - throw new Error('Run-eval org not provisioned. Call getRunEvalTestSession() first.'); - } - return runEvalUsername; -} - // Root-level hook: on any failing test that used the shared scratch org, emit orgId + pod + // UTC timestamp + error so CI logs can be correlated in Splunk. Registered once when this shared // module is first imported by the z-series NUTs. (A failing before-all hook is not caught here, but From f3f82395fab6c76c5e172136329a47568ab711d0 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Mon, 26 Oct 2026 11:53:13 -0700 Subject: [PATCH 11/15] test(nuts): fan NUTs across jobs to beat per-IP login-rate cap [do-not-merge] The 'login rate exceeded' OAuth cap is per-IP (proven: dedicated user AND dedicated org both failed in CI). Every other salesforcecli plugin with a heavy NUT suite (plugin-data 11 cmds, plugin-deploy-retrieve 15) splits its NUTs across a command matrix so each group runs on its own GitHub runner IP with its own grant budget. plugin-agent ran the entire suite serially in one job per OS, so a single IP absorbed every grant and the budget was spent before run-eval's first grant. Split the 19 NUT files into 5 groups via test:nuts:* scripts + a command matrix (5 x 2 OS = 10 jobs). run-eval is isolated on its own job for maximum headroom; z0-z5 stay in one job to preserve their shared-org create->validate->publish->preview->trace ordering. adl/mcp already skip in CI (no TARGET_ORG). The shared nut.yml accepts the 'command' input. --- .github/workflows/test.yml | 12 ++++++++++++ package.json | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04639c09..4420d9c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,18 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] + # Split the NUTs across jobs so each group runs on its own runner (its own + # egress IP). The `login rate exceeded` OAuth cap is per-IP, and running the + # whole suite serially on one IP exhausts the budget before run-eval. run-eval + # is isolated on its own job for maximum headroom; the z0-z5 pipeline stays in + # one job because its tests share one org in create->...->trace order. + command: + - 'yarn test:nuts:run-eval' + - 'yarn test:nuts:test' + - 'yarn test:nuts:generate' + - 'yarn test:nuts:z' + - 'yarn test:nuts:adl-mcp' fail-fast: false with: os: ${{ matrix.os }} + command: ${{ matrix.command }} diff --git a/package.json b/package.json index f63aa1a1..a6f3f40e 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,11 @@ "prepare": "sf-install", "test": "wireit", "test:nuts": "nyc mocha \"test/nuts/**/*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:run-eval": "nyc mocha \"test/nuts/agent.test.run-eval.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:test": "nyc mocha \"test/nuts/agent.test.nut.ts\" \"test/nuts/agent.test.create.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:generate": "nyc mocha \"test/nuts/agent.activate.nut.ts\" \"test/nuts/agent.generate.*.nut.ts\" \"test/nuts/template.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:z": "nyc mocha \"test/nuts/z*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:adl-mcp": "nyc mocha \"test/nuts/agent.adl.nut.ts\" \"test/nuts/agent.mcp.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", "test:only": "wireit", "version": "oclif readme" }, From e7c8b227b7617d867a22822d027edf69a7eca40f Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Tue, 27 Oct 2026 19:12:06 -0700 Subject: [PATCH 12/15] test(nuts): add --force-overwrite to z2 publish generate step [do-not-merge] Un-poison the this.retries(2) on the first publish test: without --force-overwrite the generate step dies with JsonAabExistsError on every retry over the bundle dir left by a failed first attempt, so the real first-attempt publish failure never surfaces in CI. With it, retries genuinely re-publish and the true error is captured. --- test/nuts/z2.agent.publish.nut.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/nuts/z2.agent.publish.nut.ts b/test/nuts/z2.agent.publish.nut.ts index ec516cc5..547ef219 100644 --- a/test/nuts/z2.agent.publish.nut.ts +++ b/test/nuts/z2.agent.publish.nut.ts @@ -119,7 +119,11 @@ describe('agent publish authoring-bundle NUTs', function () { execCmd(specCommand, { ensureExitCode: 0 }); // Step 2: Generate the authoring bundle from the spec - const generateCommand = `agent generate authoring-bundle --spec ${specPath} --name "${bundleApiName}" --api-name ${bundleApiName} --target-org ${getUsername()} --json`; + // --force-overwrite so a retry (this.retries(2)) re-generates over the bundle dir left on + // disk by a failed first attempt, instead of dying immediately with JsonAabExistsError. Without + // it, the retries are poisoned: they never reach the publish step, so the real first-attempt + // publish failure never surfaces in CI. + const generateCommand = `agent generate authoring-bundle --spec ${specPath} --name "${bundleApiName}" --api-name ${bundleApiName} --target-org ${getUsername()} --force-overwrite --json`; const generateResult = execCmd(generateCommand, { ensureExitCode: 0, }).jsonOutput?.result; From 6f5365a34f980e55109f06ba44606c6ef2906c05 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 28 Oct 2026 11:25:08 -0700 Subject: [PATCH 13/15] test(nuts): fix z2 publish verification for AiAgentDefinition output [do-not-merge] The publish NUTs asserted old behavior removed by forcedotcom/agents#310: - verifyGenAiPlannerBundleExistsOrNot checked for genAiPlannerBundles/_vN on disk, which publish no longer produces (moved to AiAgentDefinition/Bot retrieve). Replaced with verifyAgentMetadataRetrieved, which checks the current output dirs (aiAgents / aiAgentDefinitionVersions / bots). - verifyPublishedAgent was a dead no-op: a malformed SELECT SELECT query that always threw, caught by a void Promise.reject that swallowed every failure, and it checked BotVersion.DeveloperName (unrelated string) not the version. Fixed to query BotVersions.VersionNumber and assert the version count, so it actually verifies the publish for the first time. --- test/nuts/z2.agent.publish.nut.ts | 88 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/test/nuts/z2.agent.publish.nut.ts b/test/nuts/z2.agent.publish.nut.ts index 547ef219..a79afb6f 100644 --- a/test/nuts/z2.agent.publish.nut.ts +++ b/test/nuts/z2.agent.publish.nut.ts @@ -26,51 +26,52 @@ import { getAgentUsername, getTestSession, getUsername } from './shared-setup.js type BotDefinitionWithVersions = { Id: string; BotVersions: { - records: Array<{ DeveloperName: string }>; + records: Array<{ VersionNumber: number }>; }; }; const verifyPublishedAgent = async ( botApiName: string, - expectedVersion: string, + minVersionCount: number, connection: Connection ): Promise => { - let botDefinition; - try { - botDefinition = await connection.singleRecordQuery( - `SELECT SELECT Id, (SELECT DeveloperName FROM BotVersions LIMIT 10) FROM BotDefinition WHERE DeveloperName = '${botApiName}' LIMIT 1` - ); - const botVersion = botDefinition.BotVersions.records[0].DeveloperName; - expect(botVersion).to.equal(expectedVersion); - } catch { - // bot not found - void Promise.reject(new Error('Bot not published')); - } -}; - -async function verifyGenAiPlannerBundleExistsOrNot( - projectDir: string, - expectedBundleName: string, - expectedVersion: string, - shouldExist: boolean -): Promise { - const genAiPlannerBundleDir = join( - projectDir, - 'force-app', - 'main', - 'default', - 'genAiPlannerBundles', - `${expectedBundleName}_${expectedVersion}` + // singleRecordQuery throws if the agent was never published (0 records), which correctly fails + // the test. The old `SELECT SELECT ... DeveloperName` query was malformed, checked the wrong + // field (BotVersion.DeveloperName is an unrelated string like "Test_v1", not the version), and + // swallowed every failure via `void Promise.reject(...)`, so this check never actually ran. + // Each publish of the bundle adds a BotVersion, so after the Nth publish there should be at + // least N versions. We assert the COUNT (>=) rather than a specific VersionNumber because + // VersionNumber indexing isn't guaranteed to start at 1, and a --force-overwrite retry can add + // an extra version — both of which an exact check would get wrong. + const botDefinition = await connection.singleRecordQuery( + `SELECT Id, (SELECT VersionNumber FROM BotVersions ORDER BY VersionNumber) FROM BotDefinition WHERE DeveloperName = '${botApiName}' LIMIT 1` ); + expect( + botDefinition.BotVersions.records.length, + `agent "${botApiName}" should have at least ${minVersionCount} published version(s)` + ).to.be.at.least(minVersionCount); +}; - // Verify the genAiPlannerBundles directory exists - expect(existsSync(genAiPlannerBundleDir)).to.equal(shouldExist); - - if (shouldExist) { - // Verify the directory contains files - const files = readdirSync(genAiPlannerBundleDir); - expect(files.length).to.be.greaterThan(0); - } +/** + * After a successful publish the agent metadata is retrieved into the project. On modern orgs + * (API 68+) that is an `AiAgentDefinition` (dir `aiAgents`) plus an `AiAgentDefinitionVersion` + * (dir `aiAgentDefinitionVersions`); older orgs fall back to the legacy `Bot` (dir `bots`). The + * publish no longer produces the `genAiPlannerBundles/_vN` directory the old assertion + * looked for (removed with the move off the `_vN` suffix convention, forcedotcom/agents#310). We + * check for any artifact whose name starts with the agent api name, so the assertion is robust to + * the retrieve format and the exact file suffix. + */ +function verifyAgentMetadataRetrieved(projectDir: string, botApiName: string, shouldExist: boolean): void { + const defaultDir = join(projectDir, 'force-app', 'main', 'default'); + const hasEntry = (subDir: string): boolean => { + const dir = join(defaultDir, subDir); + return existsSync(dir) && readdirSync(dir).some((entry) => entry.startsWith(botApiName)); + }; + const retrieved = hasEntry('aiAgents') || hasEntry('aiAgentDefinitionVersions') || hasEntry('bots'); + expect( + retrieved, + `expected agent "${botApiName}" metadata ${shouldExist ? 'to' : 'not to'} be retrieved into the project` + ).to.equal(shouldExist); } describe('agent publish authoring-bundle NUTs', function () { @@ -150,8 +151,8 @@ describe('agent publish authoring-bundle NUTs', function () { expect(publishResult?.success).to.be.true; expect(publishResult?.botDeveloperName).to.be.a('string'); expect(publishResult?.errors).to.be.undefined; - await verifyPublishedAgent(bundleApiName, 'v1', connection); - await verifyGenAiPlannerBundleExistsOrNot(session.project.dir, bundleApiName, 'v1', true); + await verifyPublishedAgent(bundleApiName, 1, connection); + verifyAgentMetadataRetrieved(session.project.dir, bundleApiName, true); }); it('should publish a new version of an existing agent', async function () { @@ -169,8 +170,8 @@ describe('agent publish authoring-bundle NUTs', function () { expect(result?.success).to.be.true; expect(result?.botDeveloperName).to.be.a('string'); expect(result?.errors).to.be.undefined; - await verifyPublishedAgent(bundleApiName, 'v2', connection); - await verifyGenAiPlannerBundleExistsOrNot(session.project.dir, bundleApiName, 'v2', true); + await verifyPublishedAgent(bundleApiName, 2, connection); + verifyAgentMetadataRetrieved(session.project.dir, bundleApiName, true); }); it('should publish agent with skip-retrieve flag', async function () { @@ -187,10 +188,11 @@ describe('agent publish authoring-bundle NUTs', function () { ); const output = result.shellOutput.stdout; + // "Retrieve Metadata - Skipped" in stdout is the authoritative signal that --skip-retrieve + // suppressed the retrieve. A disk check can't prove v3 specifically wasn't retrieved, because + // the agent-level AiAgentDefinition from the earlier versions is already on disk. expect(output).to.include('Retrieve Metadata - Skipped'); - await verifyPublishedAgent(bundleApiName, 'v3', connection); - // skip-retrieve should not create a new version of the genAiPlannerBundle - await verifyGenAiPlannerBundleExistsOrNot(session.project.dir, bundleApiName, 'v3', false); + await verifyPublishedAgent(bundleApiName, 3, connection); }); it('should fail for invalid bundle api-name', async () => { From 97d888df888fb440eb18367ca5c3fe0ae2b4e39a Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 28 Oct 2026 15:11:17 -0700 Subject: [PATCH 14/15] test(nuts): remove stale CV5 Internal-variable rejection test [do-not-merge] CV5 asserted the server rejects a no-visibility Internal context variable (VerifiedCustomerId) on preview start. That is purely server behavior: neither the plugin (flags.ts validates only name/type/value-type) nor the SDK (scriptAgent passes `variables` straight through) enforces visibility. The test passed when written (PR #483, 2026-09-08) because the server rejected it then. The server has since changed to accept/ignore it, confirmed e2e against a live org: the command now exits 0 and returns a valid session with no 'Internal' error. The test pinned a server contract the plugin doesn't own, so it's removed. CV1-CV4/CV6 still cover the plugin's own context-variable handling. --- .../z3.agent.preview.context-variables.nut.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/test/nuts/z3.agent.preview.context-variables.nut.ts b/test/nuts/z3.agent.preview.context-variables.nut.ts index 014ff899..3abaef3f 100644 --- a/test/nuts/z3.agent.preview.context-variables.nut.ts +++ b/test/nuts/z3.agent.preview.context-variables.nut.ts @@ -37,8 +37,7 @@ import { getTestSession, getUsername } from './shared-setup.js'; * * Fixture: Willie_Resort_Manager declares test-only External vars NutProbeText (Text), * NutProbeBool (Boolean), NutProbeNum (Number), NutProbeObj (Json object), NutProbeList - * (List), plus the pre-existing Internal var VerifiedCustomerId (no visibility) used by the - * server-rejection case. + * (List). */ describe('agent preview --context-variables-json', function () { this.timeout(30 * 60 * 1000); // 30 minutes (shared setup deploys + waits on Einstein) @@ -148,15 +147,12 @@ describe('agent preview --context-variables-json', function () { expect(JSON.stringify(result.shellOutput)).to.include('Expected Name=Value'); }); - it('CV5: the server rejects an Internal variable', () => { - // VerifiedCustomerId has no visibility -> Internal -> not settable via preview. - const result = execCmd( - `agent preview start --authoring-bundle ${bundle} --simulate-actions --context-variables-json '[{"name":"VerifiedCustomerId","type":"Text","value":"x"}]' --target-org ${getUsername()} --json`, - { cwd: session.project.dir } - ); - expect(result.shellOutput.code, 'command should fail').to.not.equal(0); - expect(JSON.stringify(result.shellOutput)).to.include('Internal'); - }); + // Removed CV5 ("the server rejects an Internal variable"): it asserted a server-side + // rejection of the no-visibility Internal var VerifiedCustomerId, but that is purely server + // behavior — neither the plugin (flags.ts validates only name/type/value-type) nor the SDK + // (scriptAgent passes `variables` straight through) enforces visibility. The server stopped + // rejecting it (confirmed e2e: command exits 0 and returns a session), so the test pinned a + // server contract the plugin doesn't own. CV1–CV4/CV6 still cover the plugin's own handling. it('CV6: a value whose JSON type mismatches its declared type is rejected client-side', () => { const result = execCmd( From d0339d2654005fe119778798092ee1b589674296 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 28 Oct 2026 22:47:05 -0700 Subject: [PATCH 15/15] test(nuts): make z3 context-variables-json flag cross-platform [do-not-merge] CV1/CV2/CV3/CV6 passed the JSON payload wrapped in single quotes (--context-variables-json '[{...}]'). execCmd runs the command through shelljs, which uses bash on Unix (strips the single quotes -> clean JSON) but cmd.exe on Windows (single quotes are not special -> the literal '[...]' reaches the flag and JSON.parse rejects it). That is a test-only portability bug: the plugin parser is correct, and quote-stripping is the shell's job. Add a jsonFlag() helper that quotes per-platform (single quotes on Unix, double-quoted with escaped inner quotes on Windows) and build the payloads as JS values. Validated both forms deliver byte-identical JSON. No product change; Unix behavior unchanged. --- .../z3.agent.preview.context-variables.nut.ts | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/test/nuts/z3.agent.preview.context-variables.nut.ts b/test/nuts/z3.agent.preview.context-variables.nut.ts index 3abaef3f..36a8ebeb 100644 --- a/test/nuts/z3.agent.preview.context-variables.nut.ts +++ b/test/nuts/z3.agent.preview.context-variables.nut.ts @@ -39,6 +39,17 @@ import { getTestSession, getUsername } from './shared-setup.js'; * NutProbeBool (Boolean), NutProbeNum (Number), NutProbeObj (Json object), NutProbeList * (List). */ + +/** + * Quote a JSON value for the `--context-variables-json` flag so it survives the shell that + * execCmd runs the command through. shelljs uses bash on Unix (where single quotes keep the + * JSON's inner double quotes literal) but cmd.exe on Windows (where single quotes are NOT + * special and would reach the flag verbatim, breaking JSON.parse). Double-quoting the arg and + * escaping every inner double quote as \" delivers byte-identical clean JSON on both shells. + */ +const jsonFlag = (value: unknown): string => + process.platform === 'win32' ? `"${JSON.stringify(value).replace(/"/g, '\\"')}"` : `'${JSON.stringify(value)}'`; + describe('agent preview --context-variables-json', function () { this.timeout(30 * 60 * 1000); // 30 minutes (shared setup deploys + waits on Einstein) @@ -90,15 +101,15 @@ describe('agent preview --context-variables-json', function () { const NUM = 8_675_309; const JSON_TAG = 'NUTCV-JSON-4d21c8'; const LIST_ELEM = 'NUTCV-LIST-b58c1e'; - const payload = JSON.stringify([ + const payload = [ { name: 'NutProbeText', type: 'Text', value: TEXT }, { name: 'NutProbeNum', type: 'Number', value: NUM }, { name: 'NutProbeBool', type: 'Boolean', value: true }, { name: 'NutProbeObj', type: 'Json', value: { tag: JSON_TAG } }, { name: 'NutProbeList', type: 'List', value: [LIST_ELEM] }, - ]); + ]; - const sessionId = startSession(`--context-variables-json '${payload}'`); + const sessionId = startSession(`--context-variables-json ${jsonFlag(payload)}`); const haystack = await sendAndCollectTraces(sessionId); expect(haystack, 'Text value should reach the session').to.include(TEXT); @@ -115,7 +126,9 @@ describe('agent preview --context-variables-json', function () { const OLD = 'NUTCV-OLD-LOSES'; const NEW = 'NUTCV-NEW-WINS'; const sessionId = startSession( - `--context-variables "NutProbeText=${OLD}" --context-variables-json '[{"name":"NutProbeText","type":"Text","value":"${NEW}"}]'` + `--context-variables "NutProbeText=${OLD}" --context-variables-json ${jsonFlag([ + { name: 'NutProbeText', type: 'Text', value: NEW }, + ])}` ); const haystack = await sendAndCollectTraces(sessionId); @@ -129,7 +142,7 @@ describe('agent preview --context-variables-json', function () { this.timeout(5 * 60 * 1000); const COMMA_VALUE = 'c0,c1,c2'; const sessionId = startSession( - `--context-variables-json '[{"name":"NutProbeText","type":"Text","value":"${COMMA_VALUE}"}]'` + `--context-variables-json ${jsonFlag([{ name: 'NutProbeText', type: 'Text', value: COMMA_VALUE }])}` ); const haystack = await sendAndCollectTraces(sessionId); @@ -156,7 +169,9 @@ describe('agent preview --context-variables-json', function () { it('CV6: a value whose JSON type mismatches its declared type is rejected client-side', () => { const result = execCmd( - `agent preview start --authoring-bundle ${bundle} --simulate-actions --context-variables-json '[{"name":"NutProbeBool","type":"Boolean","value":"true"}]' --target-org ${getUsername()} --json`, + `agent preview start --authoring-bundle ${bundle} --simulate-actions --context-variables-json ${jsonFlag([ + { name: 'NutProbeBool', type: 'Boolean', value: 'true' }, + ])} --target-org ${getUsername()} --json`, { ensureExitCode: 1, cwd: session.project.dir } ); expect(JSON.stringify(result.shellOutput)).to.include('expects a boolean value');