feat(runtime): unify background service lifecycle, health and recovery

apps/presentation/dashboard/smoke/settings-target-smoke.mjs

@@ -24,6 +24,9 @@ for (const initialGoalId of [undefined, "", "unknown-goal"]) {
 const machine = render({scope: "machine", initialTab: "language"});
 assert.doesNotMatch(machine, /Target Goal|Task ownership|Automation cadence/);
 assert.match(machine, /This device/);
+assert.match(machine, /Background services/);
+assert.deepEqual(readSettingsRoute(new URL("http://localhost/chat/?settings_scope=machine&settings_tab=background")),
+  {kind: "settings", scope: "machine", tab: "background"});
 assert.deepEqual(readSettingsRoute(new URL("http://localhost/chat/?settings_scope=goal&settings_goal=shared-goal&settings_tab=ownership")),
   {kind: "settings", scope: "goal", goalId: "shared-goal", tab: "ownership"});
 assert.deepEqual(readSettingsRoute(new URL("http://localhost/chat/?goalId=background&settings_scope=goal")),

apps/presentation/dashboard/src/features/personal-workspace/background-services-settings.css

@@ -0,0 +1,20 @@
+.background-settings { display: grid; gap: 16px; max-width: 850px; }
+.background-toolbar, .background-actions, .background-task-title { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; }
+.background-toolbar > div, .background-status { display: inline-flex; align-items: center; gap: 8px; }
+.background-toolbar > .background-actions { flex-wrap: nowrap; flex-shrink: 0; }
+.background-actions .personal-secondary-action { display: inline-flex; align-items: center; justify-content: center; gap: 8px; padding: 8px 12px; margin: 0; width: auto; min-height: 44px; }
+.background-status { font-size: var(--text-body); white-space: nowrap; color: var(--pw-muted); }
+.background-status[data-state="healthy"] { color: var(--pw-green); }
+.background-status[data-state="overdue"], .background-status[data-state="retrying"] { color: var(--pw-amber); }
+.background-status[data-state="stopped"] { color: var(--pw-red); }
+.background-task-list { display: grid; gap: 12px; }
+.background-task { border: 1px solid var(--pw-line); border-radius: 12px; padding: 18px 20px; min-width: 0; }
+.background-task h2 { font-size: 1rem; font-weight: 600; margin: 0; overflow-wrap: anywhere; }
+.background-task-times { display: flex; flex-wrap: wrap; gap: 12px 32px; margin: 16px 0 0; }
+.background-task-times div { min-width: 0; }
+.background-task-times dt { color: var(--pw-muted); font-size: var(--text-caption); margin-bottom: 4px; }
+.background-task-times dd { margin: 0; overflow-wrap: anywhere; }
+.background-task details { margin-top: 14px; }
+.background-task summary { cursor: pointer; font-size: var(--text-body); color: var(--pw-muted); }
+.background-feature-link { border: 0; padding: 12px 0 0; color: var(--pw-blue-ink); background: transparent; font: inherit; cursor: pointer; }
+@media (max-width: 560px) { .background-task { padding: 16px; } .background-task-times { gap: 12px 20px; } }

apps/presentation/dashboard/src/features/personal-workspace/background-services-settings.tsx

@@ -0,0 +1,105 @@
+import {useCallback, useEffect, useRef, useState} from "react";
+import {AlertCircle, CheckCircle2, Circle, RefreshCw} from "lucide-react";
+import {z} from "zod";
+import {BACKGROUND_STATES} from "../../../../../../loopx/control_plane/runtime/background_services.ts";
+import type {BackgroundState, BackgroundServicesSnapshot} from "../../../../../../loopx/control_plane/runtime/background_services.ts";
+import {requestJson} from "../../data/chat";
+import {ContextHelp} from "./context-help";
+import {useWorkspaceI18n} from "./i18n";
+import "./background-services-settings.css";
+
+const path = "/api/chat/background-services";
+const healthSchema = z.object({state: z.enum(BACKGROUND_STATES),
+  observed_at: z.number().finite().nullable(), last_success_at: z.number().finite().nullable(),
+  last_failure_at: z.number().finite().nullable(), next_check_at: z.number().finite().nullable(),
+  failure_count: z.number().int().nonnegative().nullable(), reconcile_failed: z.boolean()});
+const snapshotSchema = z.object({ok: z.literal(true), schema_version: z.literal("background_services_v1"),
+  checked_at: z.number().finite(), supervisor: healthSchema,
+  tasks: z.array(healthSchema.extend({id: z.string().min(1), service_id: z.string().min(1)}))});
+const labels: Record<BackgroundState, [string, string]> = {
+  healthy: ["正常", "Healthy"], checking: ["正在检查", "Checking"], starting: ["正在启动", "Starting"],
+  retrying: ["退避重试中", "Backing off"], overdue: ["检查逾期", "Check overdue"],
+  stopped: ["已停止", "Stopped"], stopping: ["正在停止", "Stopping"], completed: ["已完成", "Completed"],
+  disabled: ["未启用", "Disabled"], unavailable: ["不可用", "Unavailable"], unverified: ["尚未核实", "Unverified"],
+};
+const names: Record<string, [string, string]> = {
+  "codex-observation": ["Codex 账号与额度", "Codex accounts and quota"],
+  "manager-returns": ["结果回传", "Result delivery"],
+  "manager-executions": ["排队执行恢复", "Queued execution recovery"],
+  "lark-delivery": ["Lark 消息投递", "Lark message delivery"],
+};
+const faults: BackgroundState[] = ["overdue", "retrying", "stopped", "unavailable", "unverified"];
+
+export function BackgroundServicesSettings({onOpenSettings}: {onOpenSettings?: (tab: "provider" | "lark") => void}) {
+  const {locale} = useWorkspaceI18n();
+  const zh = locale === "zh-CN", language = zh ? 0 : 1;
+  const [snapshot, setSnapshot] = useState<BackgroundServicesSnapshot | null>(null);
+  const [error, setError] = useState(false);
+  const [busy, setBusy] = useState(false);
+  const [rechecked, setRechecked] = useState(false);
+  const generation = useRef(0);
+  const mounted = useRef(true);
+  const load = useCallback(async (recheck = false) => {
+    const current = ++generation.current;
+    try {
+      const result = snapshotSchema.parse(await requestJson<unknown>(path, recheck ? {
+        method: "POST", body: JSON.stringify({operation: "recheck"}), headers: {"Content-Type": "application/json"},
+      } : undefined));
+      if (mounted.current && current === generation.current) {
+        setSnapshot(result); setError(false); if (recheck) setRechecked(true);
+      }
+    } catch {
+      if (mounted.current && current === generation.current) setError(true);
+    }
+  }, []);
+  useEffect(() => {
+    mounted.current = true;
+    void load();
+    const timer = window.setInterval(() => {if (!document.hidden) void load();}, 10_000);
+    return () => {mounted.current = false; generation.current++; window.clearInterval(timer);};
+  }, [load]);
+  const time = (value: number | null) => value === null ? "—" : new Date(value * 1000).toLocaleString(locale);
+  const status = (state: BackgroundState) => {
+    const Icon = faults.includes(state) ? AlertCircle : state === "healthy" ? CheckCircle2 : Circle;
+    return <span className="background-status" data-state={error ? "unverified" : state}><Icon size={16} aria-hidden="true" />{error ? (zh ? "上次记录：" : "Last observed: ") : ""}{labels[state][language]}</span>;
+  };
+  return <section className="background-settings" aria-label={zh ? "后台服务" : "Background services"}>
+    <div className="background-toolbar">
+      <div>{snapshot ? <>{zh ? "后台宿主" : "Background host"} {status(snapshot.supervisor.state)}</> : null}</div>
+      <div className="background-actions">
+        <ContextHelp label={zh ? "后台检查与恢复" : "Background checks and recovery"}>
+          <p>{zh ? "重新检查会恢复已启用但未运行的后台任务。授权仍由原设置管理；在途任务不会重放。检查逾期表示近期进度不足，需要查看对应模块。宿主随 LoopX 服务运行，服务进程由 App 和系统管理。" : "Recheck restores enabled background tasks that are not running. Existing settings retain authority; in-flight work is not replayed. Overdue checks need attention in the owning feature. The App and OS manage the LoopX service process."}</p>
+        </ContextHelp>
+        <button className="personal-secondary-action" disabled={busy} type="button" onClick={() => {
+          setBusy(true); setRechecked(false);
+          void load(true).finally(() => {if (mounted.current) setBusy(false);});
+        }}><RefreshCw size={15} aria-hidden="true" />{busy ? (zh ? "正在检查" : "Checking") : (zh ? "重新检查" : "Recheck")}</button>
+      </div>
+    </div>
+    {error ? <p role="alert">{zh ? "暂时无法读取后台状态。显示的记录可能已过期，请重新检查。" : "Background status is unavailable. Displayed observations may be stale; recheck to recover."}</p> : null}
+    {!snapshot && !error ? <p role="status">{zh ? "正在读取后台状态…" : "Loading background status…"}</p> : null}
+    {rechecked && !error ? <p role="status">{zh ? "已重新核对启用状态，下面显示最新观测。" : "Enabled state rechecked. Latest observations are shown below."}</p> : null}
+    {snapshot?.tasks.length === 0 ? <p>{zh ? "当前服务未登记后台任务。" : "No background tasks are registered in this service."}</p> : null}
+    <div className="background-task-list">
+      {snapshot?.tasks.map(task => <article className="background-task" key={task.id}>
+        <div className="background-task-title"><h2>{names[task.id]?.[language] ?? task.id}</h2>{status(task.state)}</div>
+        <dl className="background-task-times">
+          <div><dt>{zh ? "最近进度" : "Last progress"}</dt><dd>{time(task.observed_at)}</dd></div>
+          <div><dt>{zh ? "下次检查" : "Next check"}</dt><dd>{time(task.next_check_at)}</dd></div>
+        </dl>
+        <details><summary>{zh ? "检查记录" : "Check details"}</summary>
+          <dl className="background-task-times">
+            <div><dt>{zh ? "最近成功" : "Last success"}</dt><dd>{time(task.last_success_at)}</dd></div>
+            <div><dt>{zh ? "最近失败" : "Last failure"}</dt><dd>{time(task.last_failure_at)}</dd></div>
+            <div><dt>{zh ? "连续失败" : "Consecutive failures"}</dt><dd>{task.failure_count ?? "—"}</dd></div>
+          </dl>
+          {task.reconcile_failed ? <p>{zh ? "启用状态核对失败，将自动重试。" : "Enabled state reconciliation failed; it will retry automatically."}</p> : null}
+          {onOpenSettings && (task.service_id === "codex-accounts" || task.service_id === "lark") ? <button
+            className="background-feature-link" type="button" onClick={() => onOpenSettings(task.service_id === "lark" ? "lark" : "provider")}>
+            {task.service_id === "lark" ? (zh ? "连接设置" : "Connection settings") : (zh ? "账号设置" : "Account settings")}
+          </button> : null}
+        </details>
+      </article>)}
+    </div>
+  </section>;
+}

apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-model.ts

@@ -533,7 +533,7 @@ export type WorkspaceDrawerSelection =
   | { item: WorkspaceRun; kind: "run" }
   | { item: WorkspaceOutput; kind: "output" }
   | { item: WorkspaceActionPreview; kind: "proposal" }
-  | { goalId?: string; kind: "settings"; scope?: "machine" | "goal"; tab?: "appearance" | "capabilities" | "language" | "lark" | "machine" | "provider" | "steward" | "conversation" | "cadence" | "ownership" }
+  | { goalId?: string; kind: "settings"; scope?: "machine" | "goal"; tab?: "appearance" | "capabilities" | "language" | "lark" | "machine" | "provider" | "steward" | "conversation" | "cadence" | "ownership" | "background" }
   | {
       item: WorkspaceSchedule;
       kind: "schedule";

apps/presentation/dashboard/src/features/personal-workspace/workspace-settings-page.tsx

@@ -19,6 +19,7 @@ import { MachineConfigurationSettings } from "./machine-configuration-settings";
 import { ConfigurationBackupSettings } from "./configuration-backup-settings";
 import { OperatorCredentialSettings } from "./operator-credential-settings";
 import { CodexAccountSettings } from "./codex-account-settings";
+import { BackgroundServicesSettings } from "./background-services-settings";
 import type { PersonalWorkspaceCallbacks, WorkspaceGoal, WorkspaceGoalNotification } from "./personal-workspace-model";
 import type { WorkspaceTheme } from "./workspace-theme";
 import type { AttentionNotificationSupport } from "./attention-notifications";
@@ -30,11 +31,12 @@ const attentionNotificationSupportCopy = {
   unsupported: "settings.attentionNotificationsUnsupported",
 } as const;
 
-type WorkspaceSettingsTab = "ownership" | "steward" | "conversation" | "provider" | "machine" | "capabilities" | "cadence" | "lark" | "appearance" | "language";
+type WorkspaceSettingsTab = "ownership" | "steward" | "conversation" | "provider" | "machine" | "capabilities" | "cadence" | "lark" | "appearance" | "language" | "background";
 
 type SettingsPage = Exclude<WorkspaceSettingsTab, "machine">;
 
 const tabIcons: Record<SettingsPage, typeof Settings2> = {
+  background: ServerCog,
   ownership: KeyRound,
   appearance: Palette,
   capabilities: ServerCog,
@@ -85,6 +87,7 @@ export function WorkspaceSettingsPage({
     { key: "conversation", label: locale === "zh-CN" ? "对话与模型" : "Conversations and models" },
     { key: "provider", label: t("settings.modelProvider") },
     { key: "capabilities", label: t("settings.globalCapabilities") },
+    { key: "background", label: locale === "zh-CN" ? "后台服务" : "Background services" },
     { key: "lark", label: "Lark" },
     { key: "appearance", label: t("settings.appearance") },
     { key: "language", label: t("settings.language") },
@@ -118,6 +121,7 @@ export function WorkspaceSettingsPage({
     },
   ];
   const headings: Record<SettingsPage, { title: string }> = {
+    background: {title: locale === "zh-CN" ? "后台服务" : "Background services"},
     conversation: {title: locale === "zh-CN" ? "Goal 对话默认值" : "Goal conversation defaults"},
     ownership: {title: t("ownership.title")},
     appearance: {
@@ -146,6 +150,7 @@ export function WorkspaceSettingsPage({
   const bodyRef = useRef<HTMLElement>(null);
   useEffect(() => { if (bodyRef.current) bodyRef.current.scrollTop = 0; }, [scope, initialGoalId, tab]);
   const scopeHint = scope === "machine" ? {
+    background: "",
     steward: "",
     conversation: t("settings.effect.conversation"),
     provider: t("settings.effect.provider"),
@@ -176,7 +181,7 @@ export function WorkspaceSettingsPage({
         </fieldset>
         <label className="personal-settings-mobile-category">
           <span>{t("settings.categories")}</span>
-          <select value={tab} onChange={event => navigate(scope, initialGoalId ?? undefined, event.target.value as SettingsPage)}>
+          <select aria-label={t("settings.categories")} value={tab} onChange={event => navigate(scope, initialGoalId ?? undefined, event.target.value as SettingsPage)}>
             {tabs.map(item => <option key={item.key} value={item.key}>{item.label}</option>)}
           </select>
         </label>
@@ -237,6 +242,7 @@ export function WorkspaceSettingsPage({
           </div>
         ) : null}
 
+        {tab === "background" ? <BackgroundServicesSettings onOpenSettings={tab => navigate("machine", undefined, tab)} /> : null}
         {tab === "steward" ? <MachineConfigurationSettings
           onChanged={onChanged}
           section="steward"

apps/presentation/dashboard/src/features/personal-workspace/workspace-settings-route.ts

@@ -3,7 +3,7 @@ import type {WorkspaceDrawerSelection} from "./personal-workspace-model";
 // View state only. Configuration and permission decisions remain with each
 // existing editor and its backend; the background Goal is never a fallback.
 export type SettingsTarget = Extract<WorkspaceDrawerSelection, {kind: "settings"}>;
-const tabs = ["appearance", "capabilities", "language", "lark", "machine", "provider", "steward", "conversation", "cadence", "ownership"] as const;
+const tabs = ["appearance", "capabilities", "language", "lark", "machine", "provider", "steward", "conversation", "cadence", "ownership", "background"] as const;
 
 export function readSettingsRoute(url: URL): SettingsTarget | null {
   const scope = url.searchParams.get("settings_scope");

docs/reference/extensions.md

@@ -25,16 +25,49 @@ LoopX Core
 
 Python-hosted background loops share `loopx.extensions.background_runtime.HostedWorker`
 for idempotent thread start, cooperative close and explicitly selected failure
-retry. This is Python thread IO, not another policy or Agent supervisor. Native
+retry. Its `repeat` driver owns reconciliation repetition, next-observation
+waits and step-error backoff without rebuilding the caller's live scope. This
+is Python thread IO, not another policy or Agent supervisor. Native
 account observation, manager returns/queued-source recovery and Lark private
-delivery use it through their existing owners. A normal return is terminal;
-retry is off unless that owner explicitly enables it. A live thread does not
+delivery use it through their existing owners. A host target's normal return is
+terminal; an explicitly repeated step returning `None` uses its declared interval.
+Failure retry is off unless that owner explicitly enables it. A live thread does not
 establish provider health, and waiting, cooldown or disabled work is not a fault.
 Keep authorization, election locks, receipts, cursors/backoff and ambiguous-write
 recovery with the provider. Never use host restart to replay a model request or
 discard an in-flight journal. The standalone native account wheel retains its
 own process entrypoint and needs no Core import for account IO; when hosted by
-Core, it delegates thread lifetime to this shared host.
+Core, it delegates thread lifetime and observation cadence to this shared host.
+The same native step/health callbacks drive standalone and hosted observation;
+the small standalone wait loop remains because that distribution supports
+account IO without installing Core. Manager startup discovery is not replayed
+on a step failure, and Lark retains its existing pool, in-flight futures and
+fairness/backoff state while the scanner recovers. Shared waits never imply
+model/request replay or establish provider health from thread liveness.
+
+The Chat process registers these four loops with the process-scoped
+`BackgroundServices` lifecycle owner. It reconciles existing owner intent at
+startup and every 15 seconds, isolates owner failures, and fences shutdown
+before closing registered services. There is no additional saved enable switch,
+job queue, system daemon or permission grant. The App/OS still supervises the
+Chat process; its existing HTTP pump restores an unexpectedly exited host
+thread with a cheap, idempotent liveness hook. Recheck also restores that host,
+and never activates it from a passive read. An enabled native observer that stopped is re-created from its
+current saved policy; a live scan or a normally completed one-shot is not replayed.
+
+Settings → **Background services** reads the same content-free projection as
+`GET /api/chat/background-services`. **Recheck** uses the loopback-protected
+`POST` endpoint with `{"operation":"recheck"}` to reconcile current owner intent
+and read back observations. Reads are passive. The TypeScript owner
+`control_plane/runtime/background_services.ts` distinguishes checking, completed
+scans, backoff, overdue progress, unavailable evidence and disabled work. A
+progress deadline diagnoses silence; it never kills a thread or authorizes an
+effect retry. Successful scanning does not assert successful delivery of every
+individual request. Native observation uses its provider's elected-lock health,
+including an owner in another process; a passive Core contender cannot replace
+that evidence. Its progress timestamp is not reported as a successful quota
+sample. Per-request receipts, profile listeners and ambiguous-write recovery
+remain with their existing owners.
 
 The experimental [MemT Nano Turn host](../../packages/loopx-memt-turn/README.md)
 is a separately installed `generic-cli` execution provider. Its native memory