huangruiteng · Nov 1, 2026 · Oct 31, 2026 · Oct 31, 2026
@@ -47,6 +47,15 @@ Minimal example:
         "evidence_ref": "receipt:run-1-score",
         "origin": "official", "countable": false
       },
+      "configuration": {
+        "source_revision_id": "runner@revision-one",
+        "scorer_revision_id": "scorer@revision-one",
+        "model_id": "model-one", "budget_ms": 64800000,
+        "active_submission_cooldown_ms": 3600000,
+        "background_evaluation_interval_ms": 300000,
+        "resource_profile": "work=4cpu/16g",
+        "evidence_ref": "manifest:run-1"
+      },
       "exclude_reason": null, "rerun_reason": null
     },
     {
@@ -73,6 +82,29 @@ are null, never zero. An explicit campaign policy can supply
 `origin=authorized_failure_zero`, value zero and an evidence reference;
 such a policy value cannot claim official countability.
 
+Each selection may also carry a minimized `effort` projection with
+`duration_ms`, `token_count`, `model_request_count`, and `evidence_ref`.
+Unknown numeric fields remain null; an effort object with every metric unknown
+is rejected. The report shows both totals and per-metric coverage, so a partial
+projection cannot silently turn missing cost into zero or support an efficiency
+claim across arms with different coverage.
+
+Optional `score_samples` preserve a run's normalized score/time series. Every
+sample has strictly increasing `elapsed_ms`, a finite 0–100 `value`, the
+`normalization_id`, and an `evidence_ref`. JSON flattens the samples with their
+arm, case, and run identities for plotting. The helper does not invent missing
+points or calculate a cross-case mean curve from changing sample coverage.
+
+An optional `configuration` projection records the exact selected run's source,
+scorer, model, budget, active-submission cooldown, background-evaluation cadence,
+resource profile and evidence reference. Unknown fields remain null. The JSON
+report groups identical projections into `configuration_profiles`; Markdown
+renders one row per profile and shows evidence coverage instead of expanding
+every run reference. Multiple rows for one arm disclose within-arm heterogeneity.
+An absent projection becomes an explicit unknown profile. Configuration equality
+does not establish treatment fidelity, matched-pair qualification or causality;
+the benchmark Board and integrity owners retain those decisions.
+
 `countable` projects existing score authority; the report does not grant it.
 `exclude_reason` removes a known invalid or unresolved result from comparisons.
 Cancelled runs and live runs are always excluded. A failed run can retain a
@@ -135,8 +167,10 @@ The audit does not change score selection, normalization or countability.
   not proven causal matched comparisons. Protocol, source, resource, feedback
   and treatment fidelity still need the toolkit's matched-comparison checks.
   The helper labels every comparison descriptive and never upgrades integrity.
-- CSV provides one row per selected case/arm with score provenance and rerun
-  reasons. Spreadsheet formula prefixes are neutralized.
+- CSV provides one row per selected case/arm with score provenance, rerun
+  reasons, effort, sample count and configuration provenance. Spreadsheet
+  formula prefixes are neutralized. Full score/time samples and grouped
+  configuration profiles remain in JSON.
 
 Validation:
 
@@ -147,4 +181,4 @@ node --experimental-strip-types --test \
 
 The suite exercises the real CLI and missing versus zero scores, replacements,
 unique-case incidents, incompatible normalization, known-invalid exclusions and
-qualification contradictions.
+qualification contradictions, configuration heterogeneity and unknown fields.
@@ -14,10 +14,28 @@ type Score = {
   value: number; normalization_id: string; evidence_ref: string;
   origin: 'official' | 'authorized_failure_zero'; countable: boolean;
 };
+type Effort = {
+  duration_ms: number | null; token_count: number | null;
+  model_request_count: number | null; evidence_ref: string;
+};
+type ScoreSample = {
+  elapsed_ms: number; value: number; normalization_id: string; evidence_ref: string;
+};
+type RunConfiguration = {
+  source_revision_id: string | null;
+  scorer_revision_id: string | null;
+  model_id: string | null;
+  budget_ms: number | null;
+  active_submission_cooldown_ms: number | null;
+  background_evaluation_interval_ms: number | null;
+  resource_profile: string | null;
+  evidence_ref: string;
+};
 type Selection = {
   arm_id: string; case_id: string; run_id: string; status: Status;
   score: Score | null; protocol_id: string; exclude_reason: string | null;
   rerun_reason: Rerun | null;
+  effort?: Effort; score_samples?: ScoreSample[]; configuration?: RunConfiguration;
 };
 type Incident = {
   arm_id: string; case_id: string; run_id: string; fault: Fault;
@@ -55,6 +73,8 @@ const terminal = (s: Status) => !['planned', 'running'].includes(s);
 const key = (r: { arm_id: string; case_id: string }) => `${r.arm_id}\0${r.case_id}`;
 const mean = (v: number[]) => v.length ? v.reduce((a, b) => a + b, 0) / v.length : null;
 const digest = (v: unknown) => createHash('sha256').update(JSON.stringify(v)).digest('hex');
+const nonnegativeIntegerOrNull = (value: unknown) => value === null ||
+  (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0);
 
 export function validateInput(raw: unknown): CampaignInput {
   requireThat(raw && typeof raw === 'object', 'input must be an object');
@@ -80,6 +100,33 @@ export function validateInput(raw: unknown): CampaignInput {
       requireThat(s.origin === 'official' || (s.origin === 'authorized_failure_zero' && s.value === 0), 'score origin invalid');
       requireThat(!s.countable || (r.status === 'completed' && s.origin === 'official' && r.exclude_reason === null), 'countable score contradicts status/origin/exclusion');
     }
+    if (r.effort !== undefined) {
+      const e = r.effort;
+      requireThat(e && nonnegativeIntegerOrNull(e.duration_ms) && nonnegativeIntegerOrNull(e.token_count) &&
+        nonnegativeIntegerOrNull(e.model_request_count) && text(e.evidence_ref), 'effort must use nonnegative integers or null with evidence');
+      requireThat([e.duration_ms, e.token_count, e.model_request_count].some(v => v !== null), 'effort cannot be entirely unknown');
+    }
+    if (r.score_samples !== undefined) {
+      requireThat(Array.isArray(r.score_samples), 'score samples must be an array');
+      let previous = -1;
+      for (const sample of r.score_samples) {
+        requireThat(nonnegativeIntegerOrNull(sample?.elapsed_ms) && sample.elapsed_ms !== null &&
+          sample.elapsed_ms > previous, 'score sample elapsed time must increase');
+        requireThat(typeof sample.value === 'number' && Number.isFinite(sample.value) &&
+          sample.value >= 0 && sample.value <= 100, 'score sample must be finite normalized 0..100');
+        requireThat(token(sample.normalization_id) && text(sample.evidence_ref), 'score sample provenance required');
+        previous = sample.elapsed_ms;
+      }
+    }
+    if (r.configuration !== undefined) {
+      const c = r.configuration;
+      requireThat(c && text(c.evidence_ref), 'configuration evidence required');
+      requireThat([c.source_revision_id, c.scorer_revision_id, c.model_id].every(v => v === null || token(v)),
+        'configuration revision/model ids invalid');
+      requireThat(c.resource_profile === null || text(c.resource_profile), 'configuration resource profile invalid');
+      requireThat([c.budget_ms, c.active_submission_cooldown_ms, c.background_evaluation_interval_ms]
+        .every(nonnegativeIntegerOrNull), 'configuration numeric fields must be nonnegative integers or null');
+    }
   }
   requireThat(Array.isArray(d.incidents) && distinct(d.incidents.map(i => i.run_id)), 'incident run ids must be unique');
   for (const i of d.incidents) {
@@ -124,12 +171,26 @@ export function buildReport(raw: unknown) {
       unattributed_runtime_failures: audited.filter(r => r.outcome === 'runtime_failure' && r.attribution === 'unknown').length,
       goal_verdict_observed: audited.filter(r => !['unavailable', 'not_applicable'].includes(r.goal_verdict)).length,
     };
+    const known = <K extends keyof Effort>(field: K) => rows.filter(r => r.effort?.[field] !== null && r.effort?.[field] !== undefined);
+    const sum = (field: 'duration_ms' | 'token_count' | 'model_request_count') =>
+      known(field).reduce((total, row) => total + (row.effort![field] as number), 0);
+    const effort = {
+      duration_known_runs: known('duration_ms').length,
+      duration_ms: sum('duration_ms'),
+      token_known_runs: known('token_count').length,
+      token_count: sum('token_count'),
+      model_request_known_runs: known('model_request_count').length,
+      model_request_count: sum('model_request_count'),
+      complete_runs: rows.filter(r => r.effort && r.effort.duration_ms !== null &&
+        r.effort.token_count !== null && r.effort.model_request_count !== null).length,
+    };
     return { ...a, total: rows.length, ended: rows.filter(r => terminal(r.status)).length,
       running: rows.filter(r => r.status === 'running').length, queued: rows.filter(r => r.status === 'planned').length,
       ended_with_score: rows.filter(r => terminal(r.status) && r.score !== null).length,
       ended_without_score: rows.filter(r => terminal(r.status) && r.score === null).length,
       score_countable: rows.filter(r => r.score?.countable).length,
       first_attempt_execution: execution,
+      effort,
       product_failures_initial: counts('product', true), product_failures_ever: counts('product'),
       infrastructure_failures_ever: counts('infrastructure'), scoring_failures_ever: counts('scoring'),
       cancelled_cases_ever: counts('operator_cancelled'), unknown_failures_ever: counts('unknown'), pending_reruns: pending };
@@ -160,14 +221,48 @@ export function buildReport(raw: unknown) {
       qualified_case_ids: qualifiedCases, exclusions,
       interpretation: 'descriptive_common_case_scores_not_a_causal_or_matched_integrity_claim' };
   });
+  const score_time_series = d.selection.flatMap(row => (row.score_samples ?? []).map(sample => ({
+    arm_id: row.arm_id, case_id: row.case_id, run_id: row.run_id, ...sample,
+  })));
+  const configurationFields = ['source_revision_id', 'scorer_revision_id', 'model_id', 'budget_ms',
+    'active_submission_cooldown_ms', 'background_evaluation_interval_ms', 'resource_profile'] as const;
+  const profileGroups = new Map<string, {
+    arm_id: string; configuration_supplied: boolean; configuration: Omit<RunConfiguration, 'evidence_ref'>;
+    run_ids: string[]; case_ids: string[]; evidence_refs: string[];
+  }>();
+  for (const row of d.selection) {
+    const supplied = row.configuration !== undefined;
+    const configuration = Object.fromEntries(configurationFields.map(field =>
+      [field, row.configuration?.[field] ?? null])) as Omit<RunConfiguration, 'evidence_ref'>;
+    const profileKey = digest({arm_id: row.arm_id, configuration_supplied: supplied, configuration});
+    const group = profileGroups.get(profileKey) ?? {arm_id: row.arm_id, configuration_supplied: supplied,
+      configuration, run_ids: [], case_ids: [], evidence_refs: []};
+    group.run_ids.push(row.run_id); group.case_ids.push(row.case_id);
+    if (row.configuration) group.evidence_refs.push(row.configuration.evidence_ref);
+    profileGroups.set(profileKey, group);
+  }
+  const armOrder = new Map(d.arms.map((arm, index) => [arm.arm_id, index]));
+  const configuration_profiles = [...profileGroups.entries()].map(([profile_id, group]) => ({
+    arm_id: group.arm_id, profile_id, configuration_supplied: group.configuration_supplied,
+    ...group.configuration, run_count: group.run_ids.length, run_ids: [...group.run_ids].sort(),
+    case_ids: [...group.case_ids].sort(), evidence_refs: [...new Set(group.evidence_refs)].sort(),
+  })).sort((a, b) => (armOrder.get(a.arm_id)! - armOrder.get(b.arm_id)!) ||
+    Number(b.configuration_supplied) - Number(a.configuration_supplied) ||
+    b.run_count - a.run_count || a.profile_id.localeCompare(b.profile_id));
   return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d),
     observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection,
-    incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes };
+    configuration_profiles, score_time_series, incidents: d.incidents,
+    execution_audit: d.execution_audit ?? [], notes: d.notes };
 }
 
 type Report = ReturnType<typeof buildReport>;
 const cell = (v: unknown) => String(v ?? '—').replaceAll('|', '\\|').replace(/[\r\n]+/g, ' ').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
 const number = (v: number | null) => v === null ? '—' : v.toFixed(2);
+const known = (v: unknown) => v === null || v === undefined ? '未知' : v;
+const hours = (v: number | null) => v === null ? '未知' : (v / 3_600_000).toFixed(2);
+const seconds = (v: number | null) => v === null ? '未知' : (v / 1000).toFixed(0);
+const coveredTotal = (coverage: number, value: number, scale = 1) =>
+  coverage === 0 ? null : scale === 1 ? value : (value / scale).toFixed(2);
 function table(headers: string[], rows: unknown[][]) {
   return ['| ' + headers.map(cell).join(' | ') + ' |', '| ' + headers.map(() => '---').join(' | ') + ' |',
     ...rows.map(r => '| ' + r.map(cell).join(' | ') + ' |')].join('\n');
@@ -185,17 +280,22 @@ export function renderMarkdown(r: Report): string {
     })) +
     '\n\n以下仅计已记录事件，缺失记录不证明没有故障，不能据此直接比较故障率。产品归因不是所有运行异常的总数。历史按每组独立题目去重；待补跑包含正在补跑。\n\n' +
     table(['实验组','首次已归因产品故障','历史已归因产品故障','历史已归因基础设施故障','历史归因待查','历史调度取消','基础设施待补跑','评分待补跑','取消待补跑','其他待重做'], r.summary.map(a => [a.label,a.product_failures_initial,a.product_failures_ever,a.infrastructure_failures_ever,a.unknown_failures_ever,a.cancelled_cases_ever,a.pending_reruns.infrastructure,a.pending_reruns.scoring,a.pending_reruns.operator_cancelled,(a.pending_reruns.comparison_change ?? 0)+(a.pending_reruns.integrity_review ?? 0)+(a.pending_reruns.product ?? 0)+(a.pending_reruns.unknown ?? 0)])) +
+    '\n\n成本只汇总调用方提供并带证据的运行时长、token 和模型请求数；未知不按零计。各组覆盖数不同，不能直接用总量判断效率。\n\n' +
+    table(['实验组','时长覆盖','总运行时长(h)','Token覆盖','总Token','请求数覆盖','模型请求数','三项完整'], r.summary.map(a => [a.label,`${a.effort.duration_known_runs}/${a.total}`,coveredTotal(a.effort.duration_known_runs,a.effort.duration_ms,3_600_000),`${a.effort.token_known_runs}/${a.total}`,coveredTotal(a.effort.token_known_runs,a.effort.token_count),`${a.effort.model_request_known_runs}/${a.total}`,coveredTotal(a.effort.model_request_known_runs,a.effort.model_request_count),a.effort.complete_runs])) +
+    '\n\n## 运行配置矩阵\n\n每行是一组实际出现的配置；同一组出现多行表示该组内部配置并不一致。未知值保持未知。配置相同也不自动证明严格配对、处理忠实度或因果可比性。\n\n' +
+    table(['实验组','运行数','Source','Scorer','Model','预算(h)','主动提交冷却(s)','后台评测(s)','资源档位','证据覆盖'],r.configuration_profiles.map(p => [labels.get(p.arm_id),p.run_count,known(p.source_revision_id),known(p.scorer_revision_id),known(p.model_id),hours(p.budget_ms),seconds(p.active_submission_cooldown_ms),seconds(p.background_evaluation_interval_ms),known(p.resource_profile),`${p.configuration_supplied ? p.run_count : 0}/${p.run_count}`])) +
     '\n\n均分使用相同题目交集、相同归一化规则，尺度 0–100。未出分不填 0；授权的故障判零单独保留来源。以下为描述性比较；源版本、协议、资源差异和完整性复核仍需独立审查。\n\n' +
     table(['对比范围','共同题数',...r.summary.map(a => a.label),'第二组−第一组','共同题中双方/各方可计数'],r.comparisons.map(c => [c.arms.map(a => labels.get(a)).join(' / '),c.n,...r.summary.map(a => number(c.means[a.arm_id] ?? null)),number(c.delta_second_minus_first),c.qualified_common_n])) +
     '\n\n' + r.notes.map(n => '- ' + cell(n)).join('\n') + '\n\n## 共同题目与排除记录\n\n' +
     r.comparisons.map(c => `- ${c.arms.map(a => labels.get(a)).join(' / ')}：${c.case_ids.join('、') || '无'}。`).join('\n') +
-    '\n\n逐题状态、每项分数来源、排除原因及历史故障均保留在同名 JSON；CSV 每行是一道题的一个组。\n';
+    `\n\n分数/时间序列：${r.score_time_series.length} 个带来源的采样点，覆盖 ${new Set(r.score_time_series.map(s => s.run_id)).size} 个运行。采样覆盖不同，不从缺失点补零，也不自动计算跨题均值曲线；同名 JSON 保留可绘图序列。\n\n` +
+    '逐题状态、每项分数来源、排除原因及历史故障均保留在同名 JSON；CSV 每行是一道题的一个组。\n';
 }
 export function renderCsv(r: Report): string {
   // Neutralize spreadsheet formulas in caller-controlled labels and ids.
   const q = (v: unknown) => { let s = String(v ?? ''); if (/^\s*[=+\-@]/.test(s)) s = "'" + s; return '"' + s.replaceAll('"','""') + '"'; };
-  return [['arm_id','case_id','run_id','status','score_0_100','score_origin','normalization_id','countable','rerun_reason','exclude_reason','evidence_ref'],
-    ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n';
+  return [['arm_id','case_id','run_id','status','score_0_100','score_origin','normalization_id','countable','rerun_reason','exclude_reason','score_evidence_ref','duration_ms','token_count','model_request_count','effort_evidence_ref','score_sample_count','source_revision_id','scorer_revision_id','model_id','budget_ms','active_submission_cooldown_ms','background_evaluation_interval_ms','resource_profile','configuration_evidence_ref'],
+    ...r.selection.map(s => [s.arm_id,s.case_id,s.run_id,s.status,s.score?.value,s.score?.origin,s.score?.normalization_id,s.score?.countable,s.rerun_reason,s.exclude_reason,s.score?.evidence_ref,s.effort?.duration_ms,s.effort?.token_count,s.effort?.model_request_count,s.effort?.evidence_ref,s.score_samples?.length ?? 0,s.configuration?.source_revision_id,s.configuration?.scorer_revision_id,s.configuration?.model_id,s.configuration?.budget_ms,s.configuration?.active_submission_cooldown_ms,s.configuration?.background_evaluation_interval_ms,s.configuration?.resource_profile,s.configuration?.evidence_ref])].map(row => row.map(q).join(',')).join('\n')+'\n';
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {