diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index 883cd64a86..fd8807704e 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -47,6 +47,15 @@ Minimal example: "evidence_ref": "receipt:run-1-score", "origin": "official", "countable": false }, + "configuration": { + "source_revision_id": "runner@revision-one", + "scorer_revision_id": "scorer@revision-one", + "model_id": "model-one", "budget_ms": 64800000, + "active_submission_cooldown_ms": 3600000, + "background_evaluation_interval_ms": 300000, + "resource_profile": "work=4cpu/16g", + "evidence_ref": "manifest:run-1" + }, "exclude_reason": null, "rerun_reason": null }, { @@ -73,6 +82,29 @@ are null, never zero. An explicit campaign policy can supply `origin=authorized_failure_zero`, value zero and an evidence reference; such a policy value cannot claim official countability. +Each selection may also carry a minimized `effort` projection with +`duration_ms`, `token_count`, `model_request_count`, and `evidence_ref`. +Unknown numeric fields remain null; an effort object with every metric unknown +is rejected. The report shows both totals and per-metric coverage, so a partial +projection cannot silently turn missing cost into zero or support an efficiency +claim across arms with different coverage. + +Optional `score_samples` preserve a run's normalized score/time series. Every +sample has strictly increasing `elapsed_ms`, a finite 0–100 `value`, the +`normalization_id`, and an `evidence_ref`. JSON flattens the samples with their +arm, case, and run identities for plotting. The helper does not invent missing +points or calculate a cross-case mean curve from changing sample coverage. + +An optional `configuration` projection records the exact selected run's source, +scorer, model, budget, active-submission cooldown, background-evaluation cadence, +resource profile and evidence reference. Unknown fields remain null. The JSON +report groups identical projections into `configuration_profiles`; Markdown +renders one row per profile and shows evidence coverage instead of expanding +every run reference. Multiple rows for one arm disclose within-arm heterogeneity. +An absent projection becomes an explicit unknown profile. Configuration equality +does not establish treatment fidelity, matched-pair qualification or causality; +the benchmark Board and integrity owners retain those decisions. + `countable` projects existing score authority; the report does not grant it. `exclude_reason` removes a known invalid or unresolved result from comparisons. Cancelled runs and live runs are always excluded. A failed run can retain a @@ -135,8 +167,10 @@ The audit does not change score selection, normalization or countability. not proven causal matched comparisons. Protocol, source, resource, feedback and treatment fidelity still need the toolkit's matched-comparison checks. The helper labels every comparison descriptive and never upgrades integrity. -- CSV provides one row per selected case/arm with score provenance and rerun - reasons. Spreadsheet formula prefixes are neutralized. +- CSV provides one row per selected case/arm with score provenance, rerun + reasons, effort, sample count and configuration provenance. Spreadsheet + formula prefixes are neutralized. Full score/time samples and grouped + configuration profiles remain in JSON. Validation: @@ -147,4 +181,4 @@ node --experimental-strip-types --test \ The suite exercises the real CLI and missing versus zero scores, replacements, unique-case incidents, incompatible normalization, known-invalid exclusions and -qualification contradictions. +qualification contradictions, configuration heterogeneity and unknown fields. diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 67c948a7b1..1935a35939 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -14,10 +14,28 @@ type Score = { value: number; normalization_id: string; evidence_ref: string; origin: 'official' | 'authorized_failure_zero'; countable: boolean; }; +type Effort = { + duration_ms: number | null; token_count: number | null; + model_request_count: number | null; evidence_ref: string; +}; +type ScoreSample = { + elapsed_ms: number; value: number; normalization_id: string; evidence_ref: string; +}; +type RunConfiguration = { + source_revision_id: string | null; + scorer_revision_id: string | null; + model_id: string | null; + budget_ms: number | null; + active_submission_cooldown_ms: number | null; + background_evaluation_interval_ms: number | null; + resource_profile: string | null; + evidence_ref: string; +}; type Selection = { arm_id: string; case_id: string; run_id: string; status: Status; score: Score | null; protocol_id: string; exclude_reason: string | null; rerun_reason: Rerun | null; + effort?: Effort; score_samples?: ScoreSample[]; configuration?: RunConfiguration; }; type Incident = { arm_id: string; case_id: string; run_id: string; fault: Fault; @@ -55,6 +73,8 @@ const terminal = (s: Status) => !['planned', 'running'].includes(s); const key = (r: { arm_id: string; case_id: string }) => `${r.arm_id}\0${r.case_id}`; const mean = (v: number[]) => v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; const digest = (v: unknown) => createHash('sha256').update(JSON.stringify(v)).digest('hex'); +const nonnegativeIntegerOrNull = (value: unknown) => value === null || + (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0); export function validateInput(raw: unknown): CampaignInput { requireThat(raw && typeof raw === 'object', 'input must be an object'); @@ -80,6 +100,33 @@ export function validateInput(raw: unknown): CampaignInput { requireThat(s.origin === 'official' || (s.origin === 'authorized_failure_zero' && s.value === 0), 'score origin invalid'); requireThat(!s.countable || (r.status === 'completed' && s.origin === 'official' && r.exclude_reason === null), 'countable score contradicts status/origin/exclusion'); } + if (r.effort !== undefined) { + const e = r.effort; + requireThat(e && nonnegativeIntegerOrNull(e.duration_ms) && nonnegativeIntegerOrNull(e.token_count) && + nonnegativeIntegerOrNull(e.model_request_count) && text(e.evidence_ref), 'effort must use nonnegative integers or null with evidence'); + requireThat([e.duration_ms, e.token_count, e.model_request_count].some(v => v !== null), 'effort cannot be entirely unknown'); + } + if (r.score_samples !== undefined) { + requireThat(Array.isArray(r.score_samples), 'score samples must be an array'); + let previous = -1; + for (const sample of r.score_samples) { + requireThat(nonnegativeIntegerOrNull(sample?.elapsed_ms) && sample.elapsed_ms !== null && + sample.elapsed_ms > previous, 'score sample elapsed time must increase'); + requireThat(typeof sample.value === 'number' && Number.isFinite(sample.value) && + sample.value >= 0 && sample.value <= 100, 'score sample must be finite normalized 0..100'); + requireThat(token(sample.normalization_id) && text(sample.evidence_ref), 'score sample provenance required'); + previous = sample.elapsed_ms; + } + } + if (r.configuration !== undefined) { + const c = r.configuration; + requireThat(c && text(c.evidence_ref), 'configuration evidence required'); + requireThat([c.source_revision_id, c.scorer_revision_id, c.model_id].every(v => v === null || token(v)), + 'configuration revision/model ids invalid'); + requireThat(c.resource_profile === null || text(c.resource_profile), 'configuration resource profile invalid'); + requireThat([c.budget_ms, c.active_submission_cooldown_ms, c.background_evaluation_interval_ms] + .every(nonnegativeIntegerOrNull), 'configuration numeric fields must be nonnegative integers or null'); + } } requireThat(Array.isArray(d.incidents) && distinct(d.incidents.map(i => i.run_id)), 'incident run ids must be unique'); for (const i of d.incidents) { @@ -124,12 +171,26 @@ export function buildReport(raw: unknown) { unattributed_runtime_failures: audited.filter(r => r.outcome === 'runtime_failure' && r.attribution === 'unknown').length, goal_verdict_observed: audited.filter(r => !['unavailable', 'not_applicable'].includes(r.goal_verdict)).length, }; + const known = (field: K) => rows.filter(r => r.effort?.[field] !== null && r.effort?.[field] !== undefined); + const sum = (field: 'duration_ms' | 'token_count' | 'model_request_count') => + known(field).reduce((total, row) => total + (row.effort![field] as number), 0); + const effort = { + duration_known_runs: known('duration_ms').length, + duration_ms: sum('duration_ms'), + token_known_runs: known('token_count').length, + token_count: sum('token_count'), + model_request_known_runs: known('model_request_count').length, + model_request_count: sum('model_request_count'), + complete_runs: rows.filter(r => r.effort && r.effort.duration_ms !== null && + r.effort.token_count !== null && r.effort.model_request_count !== null).length, + }; return { ...a, total: rows.length, ended: rows.filter(r => terminal(r.status)).length, running: rows.filter(r => r.status === 'running').length, queued: rows.filter(r => r.status === 'planned').length, ended_with_score: rows.filter(r => terminal(r.status) && r.score !== null).length, ended_without_score: rows.filter(r => terminal(r.status) && r.score === null).length, score_countable: rows.filter(r => r.score?.countable).length, first_attempt_execution: execution, + effort, product_failures_initial: counts('product', true), product_failures_ever: counts('product'), infrastructure_failures_ever: counts('infrastructure'), scoring_failures_ever: counts('scoring'), cancelled_cases_ever: counts('operator_cancelled'), unknown_failures_ever: counts('unknown'), pending_reruns: pending }; @@ -160,14 +221,48 @@ export function buildReport(raw: unknown) { qualified_case_ids: qualifiedCases, exclusions, interpretation: 'descriptive_common_case_scores_not_a_causal_or_matched_integrity_claim' }; }); + const score_time_series = d.selection.flatMap(row => (row.score_samples ?? []).map(sample => ({ + arm_id: row.arm_id, case_id: row.case_id, run_id: row.run_id, ...sample, + }))); + const configurationFields = ['source_revision_id', 'scorer_revision_id', 'model_id', 'budget_ms', + 'active_submission_cooldown_ms', 'background_evaluation_interval_ms', 'resource_profile'] as const; + const profileGroups = new Map; + run_ids: string[]; case_ids: string[]; evidence_refs: string[]; + }>(); + for (const row of d.selection) { + const supplied = row.configuration !== undefined; + const configuration = Object.fromEntries(configurationFields.map(field => + [field, row.configuration?.[field] ?? null])) as Omit; + const profileKey = digest({arm_id: row.arm_id, configuration_supplied: supplied, configuration}); + const group = profileGroups.get(profileKey) ?? {arm_id: row.arm_id, configuration_supplied: supplied, + configuration, run_ids: [], case_ids: [], evidence_refs: []}; + group.run_ids.push(row.run_id); group.case_ids.push(row.case_id); + if (row.configuration) group.evidence_refs.push(row.configuration.evidence_ref); + profileGroups.set(profileKey, group); + } + const armOrder = new Map(d.arms.map((arm, index) => [arm.arm_id, index])); + const configuration_profiles = [...profileGroups.entries()].map(([profile_id, group]) => ({ + arm_id: group.arm_id, profile_id, configuration_supplied: group.configuration_supplied, + ...group.configuration, run_count: group.run_ids.length, run_ids: [...group.run_ids].sort(), + case_ids: [...group.case_ids].sort(), evidence_refs: [...new Set(group.evidence_refs)].sort(), + })).sort((a, b) => (armOrder.get(a.arm_id)! - armOrder.get(b.arm_id)!) || + Number(b.configuration_supplied) - Number(a.configuration_supplied) || + b.run_count - a.run_count || a.profile_id.localeCompare(b.profile_id)); return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d), observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection, - incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes }; + configuration_profiles, score_time_series, incidents: d.incidents, + execution_audit: d.execution_audit ?? [], notes: d.notes }; } type Report = ReturnType; const cell = (v: unknown) => String(v ?? '—').replaceAll('|', '\\|').replace(/[\r\n]+/g, ' ').replaceAll('<', '<').replaceAll('>', '>'); const number = (v: number | null) => v === null ? '—' : v.toFixed(2); +const known = (v: unknown) => v === null || v === undefined ? '未知' : v; +const hours = (v: number | null) => v === null ? '未知' : (v / 3_600_000).toFixed(2); +const seconds = (v: number | null) => v === null ? '未知' : (v / 1000).toFixed(0); +const coveredTotal = (coverage: number, value: number, scale = 1) => + coverage === 0 ? null : scale === 1 ? value : (value / scale).toFixed(2); function table(headers: string[], rows: unknown[][]) { return ['| ' + headers.map(cell).join(' | ') + ' |', '| ' + headers.map(() => '---').join(' | ') + ' |', ...rows.map(r => '| ' + r.map(cell).join(' | ') + ' |')].join('\n'); @@ -185,17 +280,22 @@ export function renderMarkdown(r: Report): string { })) + '\n\n以下仅计已记录事件，缺失记录不证明没有故障，不能据此直接比较故障率。产品归因不是所有运行异常的总数。历史按每组独立题目去重；待补跑包含正在补跑。\n\n' + table(['实验组','首次已归因产品故障','历史已归因产品故障','历史已归因基础设施故障','历史归因待查','历史调度取消','基础设施待补跑','评分待补跑','取消待补跑','其他待重做'], r.summary.map(a => [a.label,a.product_failures_initial,a.product_failures_ever,a.infrastructure_failures_ever,a.unknown_failures_ever,a.cancelled_cases_ever,a.pending_reruns.infrastructure,a.pending_reruns.scoring,a.pending_reruns.operator_cancelled,(a.pending_reruns.comparison_change ?? 0)+(a.pending_reruns.integrity_review ?? 0)+(a.pending_reruns.product ?? 0)+(a.pending_reruns.unknown ?? 0)])) + + '\n\n成本只汇总调用方提供并带证据的运行时长、token 和模型请求数；未知不按零计。各组覆盖数不同，不能直接用总量判断效率。\n\n' + + table(['实验组','时长覆盖','总运行时长(h)','Token覆盖','总Token','请求数覆盖','模型请求数','三项完整'], r.summary.map(a => [a.label,`${a.effort.duration_known_runs}/${a.total}`,coveredTotal(a.effort.duration_known_runs,a.effort.duration_ms,3_600_000),`${a.effort.token_known_runs}/${a.total}`,coveredTotal(a.effort.token_known_runs,a.effort.token_count),`${a.effort.model_request_known_runs}/${a.total}`,coveredTotal(a.effort.model_request_known_runs,a.effort.model_request_count),a.effort.complete_runs])) + + '\n\n## 运行配置矩阵\n\n每行是一组实际出现的配置；同一组出现多行表示该组内部配置并不一致。未知值保持未知。配置相同也不自动证明严格配对、处理忠实度或因果可比性。\n\n' + + table(['实验组','运行数','Source','Scorer','Model','预算(h)','主动提交冷却(s)','后台评测(s)','资源档位','证据覆盖'],r.configuration_profiles.map(p => [labels.get(p.arm_id),p.run_count,known(p.source_revision_id),known(p.scorer_revision_id),known(p.model_id),hours(p.budget_ms),seconds(p.active_submission_cooldown_ms),seconds(p.background_evaluation_interval_ms),known(p.resource_profile),`${p.configuration_supplied ? p.run_count : 0}/${p.run_count}`])) + '\n\n均分使用相同题目交集、相同归一化规则，尺度 0–100。未出分不填 0；授权的故障判零单独保留来源。以下为描述性比较；源版本、协议、资源差异和完整性复核仍需独立审查。\n\n' + table(['对比范围','共同题数',...r.summary.map(a => a.label),'第二组−第一组','共同题中双方/各方可计数'],r.comparisons.map(c => [c.arms.map(a => labels.get(a)).join(' / '),c.n,...r.summary.map(a => number(c.means[a.arm_id] ?? null)),number(c.delta_second_minus_first),c.qualified_common_n])) + '\n\n' + r.notes.map(n => '- ' + cell(n)).join('\n') + '\n\n## 共同题目与排除记录\n\n' + r.comparisons.map(c => `- ${c.arms.map(a => labels.get(a)).join(' / ')}：${c.case_ids.join('、') || '无'}。`).join('\n') + - '\n\n逐题状态、每项分数来源、排除原因及历史故障均保留在同名 JSON；CSV 每行是一道题的一个组。\n'; + `\n\n分数/时间序列：${r.score_time_series.length} 个带来源的采样点，覆盖 ${new Set(r.score_time_series.map(s => s.run_id)).size} 个运行。采样覆盖不同，不从缺失点补零，也不自动计算跨题均值曲线；同名 JSON 保留可绘图序列。\n\n` + + '逐题状态、每项分数来源、排除原因及历史故障均保留在同名 JSON；CSV 每行是一道题的一个组。\n'; } export function renderCsv(r: Report): string { // Neutralize spreadsheet formulas in caller-controlled labels and ids. const q = (v: unknown) => { let s = String(v ?? ''); if (/^\s*[=+\-@]/.test(s)) s = "'" + s; return '"' + s.replaceAll('"','""') + '"'; }; - return [['arm_id','case_id','run_id','status','score_0_100','score_origin','normalization_id','countable','rerun_reason','exclude_reason','evidence_ref'], - ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n'; + return [['arm_id','case_id','run_id','status','score_0_100','score_origin','normalization_id','countable','rerun_reason','exclude_reason','score_evidence_ref','duration_ms','token_count','model_request_count','effort_evidence_ref','score_sample_count','source_revision_id','scorer_revision_id','model_id','budget_ms','active_submission_cooldown_ms','background_evaluation_interval_ms','resource_profile','configuration_evidence_ref'], + ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref,s.effort?.duration_ms,s.effort?.token_count,s.effort?.model_request_count,s.effort?.evidence_ref,s.score_samples?.length ?? 0,s.configuration?.source_revision_id,s.configuration?.scorer_revision_id,s.configuration?.model_id,s.configuration?.budget_ms,s.configuration?.active_submission_cooldown_ms,s.configuration?.background_evaluation_interval_ms,s.configuration?.resource_profile,s.configuration?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n'; } if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 6c49f2e1b7..57751feb0b 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -113,3 +113,60 @@ test('same-attempt recovered error is not a terminal failure', () => { const e=buildReport(d).summary[0].first_attempt_execution; assert.equal(e.outcomes.normal_end,1);assert.equal(e.outcomes.runtime_failure,0); }); + +test('effort and score-time samples preserve coverage and unknowns', () => { + const d=fixture(); + d.selection[0].effort={duration_ms:3_600_000,token_count:1200,model_request_count:null,evidence_ref:'usage:a-one'}; + d.selection[0].score_samples=[ + {elapsed_ms:60_000,value:10,normalization_id:'scale-one',evidence_ref:'score:first'}, + {elapsed_ms:3_600_000,value:40,normalization_id:'scale-one',evidence_ref:'score:last'}, + ]; + const r=buildReport(d); + assert.deepEqual(r.summary[0].effort,{duration_known_runs:1,duration_ms:3_600_000, + token_known_runs:1,token_count:1200,model_request_known_runs:0,model_request_count:0,complete_runs:0}); + assert.equal(r.score_time_series.length,2);assert.equal(r.score_time_series[1].run_id,'a-one'); + assert.match(renderMarkdown(r),/1\.00/);assert.match(renderMarkdown(r),/未知不按零计/); + assert.match(renderMarkdown(r),/0\/3 \| — \| 0\/3 \| —/); + assert.match(renderCsv(r),/3600000/); +}); + +test('effort and score-time samples reject malformed measurements', () => { + const d=fixture(); + d.selection[0].effort={duration_ms:null,token_count:null,model_request_count:null,evidence_ref:'usage:none'}; + assert.throws(()=>buildReport(d),/entirely unknown/); + d.selection[0].effort={duration_ms:1,token_count:2,model_request_count:3,evidence_ref:'usage:ok'}; + d.selection[0].score_samples=[ + {elapsed_ms:2,value:10,normalization_id:'scale-one',evidence_ref:'score:2'}, + {elapsed_ms:1,value:20,normalization_id:'scale-one',evidence_ref:'score:1'}, + ]; + assert.throws(()=>buildReport(d),/elapsed time must increase/); +}); + +test('configuration profiles preserve unknowns and split heterogeneous runs', () => { + const d=fixture(); + d.selection[0].configuration={source_revision_id:'source@one',scorer_revision_id:'scorer@one', + model_id:'model-one',budget_ms:64_800_000,active_submission_cooldown_ms:3_600_000, + background_evaluation_interval_ms:300_000,resource_profile:'cpu4-mem16g',evidence_ref:'manifest:a-one'}; + d.selection[1].configuration={source_revision_id:'source@two',scorer_revision_id:'scorer@one', + model_id:'model-one',budget_ms:64_800_000,active_submission_cooldown_ms:3_600_000, + background_evaluation_interval_ms:null,resource_profile:null,evidence_ref:'manifest:a-two'}; + const r=buildReport(d); + const a=r.configuration_profiles.filter(p=>p.arm_id==='a'); + assert.equal(a.length,3);assert.deepEqual(a.map(p=>p.run_count).sort(),[1,1,1]); + assert.equal(a.find(p=>p.source_revision_id==='source@two')?.background_evaluation_interval_ms,null); + assert.equal(a.find(p=>p.configuration_supplied===false)?.run_count,1); + const markdown=renderMarkdown(r); + assert.match(markdown,/运行配置矩阵/);assert.match(markdown,/未知/);assert.match(markdown,/同一组出现多行/); + assert.match(renderCsv(r),/source_revision_id/);assert.match(renderCsv(r),/manifest:a-one/); +}); + +test('configuration requires typed public-safe fields and evidence', () => { + const d=fixture(); + d.selection[0].configuration={source_revision_id:'source',scorer_revision_id:'scorer',model_id:'model', + budget_ms:1,active_submission_cooldown_ms:2,background_evaluation_interval_ms:3, + resource_profile:'cpu4',evidence_ref:''}; + assert.throws(()=>buildReport(d),/configuration evidence/); + d.selection[0].configuration.evidence_ref='manifest:one'; + d.selection[0].configuration.budget_ms=-1; + assert.throws(()=>buildReport(d),/configuration numeric/); +});