feat(sqlite): use WAL with synchronous NORMAL for file databases

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+- A SQLite file database now uses write-ahead logging (WAL) with
+  `synchronous = NORMAL`, the settings that the Ruby gem gets from Active
+  Record. Before, each commit made about four `fsync` calls, and
+  `node:sqlite` blocked the event loop while it waited for the disk; on a slow
+  disk this stopped timers, heartbeats, and lease renewals. A process crash
+  loses no commit. A power loss or an operating system crash can now lose the
+  last commits, and the database stays consistent. Pass `synchronous: "full"`
+  to flush each commit before it returns. The adapter switches an existing file
+  to WAL on its first access, with the same retries as a transaction. WAL keeps
+  `-wal` and `-shm` files next to the database.
 - The short answer in `docs/virtual-actors.md` no longer states that version
   1.0 makes no claim of measured scale. It links `docs/versioning.md`, the one
   page that states the limits of version 1.0. Version 1.0.5 missed this

docs/api.md

@@ -776,7 +776,10 @@ and `SyncTimeoutWaitingOn` type timeout diagnostics. See
 
 - `sqlite(options)`: construct `SQLiteDatabase`.
 - `SQLiteDatabase`: `Database` implementation and `close()` owner.
-- `SQLiteDatabaseOptions`: path, busy timeout, and lock retry options.
+- `SQLiteDatabaseOptions`: path, busy timeout, lock retry, and `synchronous`
+  options.
+- `SQLiteSynchronous`: `"normal" | "full"`, the `synchronous` level of a file
+  database.
 
 ## `solid-objects/database/sqlite-wasm`
 

docs/configuration.md

@@ -167,11 +167,30 @@ IDs, while PostgreSQL callers should use `RETURNING` with `get()`.
 
 ### SQLite
 
-`sqlite({ path, timeoutMilliseconds = 5_000, lockRetryAttempts = 10 })` uses
-Node's built-in `node:sqlite`. `lockRetryAttempts` is a positive integer. The
-timeout controls SQLite's busy timeout; transaction acquisition retries use
+`sqlite({ path, timeoutMilliseconds = 5_000, lockRetryAttempts = 10, synchronous = "normal" })`
+uses Node's built-in `node:sqlite`. `lockRetryAttempts` is a positive integer.
+The timeout controls SQLite's busy timeout; transaction acquisition retries use
 short capped backoff only when no synchronous deadline is active.
 
+A file database uses write-ahead logging (WAL). On its first access, the
+adapter sets `journal_mode = WAL` and the `synchronous` level, with the same
+retries as a transaction. WAL keeps two more files next to the database:
+`<path>-wal` and `<path>-shm`. Committed data can be in the `-wal` file, so a
+copy of the database file alone can miss it. To back up a database that is in
+use, run `VACUUM INTO '<backup path>'` or use the SQLite backup command. Copy
+the files only when no process has the database open. An in-memory database
+(`":memory:"`) keeps its memory journal.
+
+`synchronous` is `"normal"` or `"full"`:
+
+- `"normal"`, the default, is the level that the Ruby gem gets from Active
+  Record. A commit makes almost no `fsync` call. A process crash loses no
+  commit. A power loss or an operating system crash can lose the last commits,
+  but the database stays consistent.
+- `"full"` flushes each commit to disk before the commit returns. A commit
+  costs about one `fsync` call, and `node:sqlite` blocks the event loop while
+  it waits for the disk.
+
 ### PostgreSQL
 
 `postgresql(options)` accepts:

docs/parity.md

@@ -101,6 +101,7 @@ Ruby publishes RBS contracts with native Ruby field names; this does not change
 | Capability                  | Status | TypeScript shape or remaining work                                                                                                                                                                                                                                                                                                                                                    |
 | --------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | SQLite                      | Native | Uses built-in `node:sqlite`, serialized process-local access, bounded transient writer retries, foreign keys, strict tables, database time, and deadline-bounded access and lock waits. Ruby uses a yielding busy handler for background writes so a waiting thread lets the lock holder commit, including on Rails 7.1/7.2; both preserve synchronous deadlines.                     |
+| SQLite durability settings  | Native | File databases use WAL with synchronous NORMAL in both runtimes. Ruby gets them from Active Record; the Node adapter sets them on first access, and `synchronous: "full"` is optional.                                                                                                                                                                                                |
 | PostgreSQL                  | Native | Optional `pg` 8.23 peer, bounded pooling, 64-bit schema, row-locked sequences, server checks, and deadline-bounded pool, statement, and lock waits.                                                                                                                                                                                                                                   |
 | MySQL                       | Native | Optional `mysql2` 3.23 peer, bounded pooling, InnoDB schema, row-locked sequences, scoped deadlock retry, and deadline-bounded pool, query, and lock waits. Ruby also tests a second client, `trilogy`; Node has no comparable second MySQL client, so only `mysql2` is tracked here.                                                                                                 |
 | Durable polling fallback    | Native | Every role progresses without a notification service. Effects, reminders, and broadcasts use canonical ordered polling indexes; PostgreSQL and MySQL lock only the selected row with `FOR UPDATE SKIP LOCKED`. Broadcasts and reminders compare separate available and stale-recovery probes, preserve the oldest-first choice, and retry past candidates locked by another claimant. |

src/database/sqlite.ts

@@ -9,10 +9,13 @@ import {
 import { DatabaseDeadlineExceeded } from "../errors.js"
 import { databaseTransactionActive, withDatabaseTransaction } from "./transaction-context.js"
 
+export type SQLiteSynchronous = "normal" | "full"
+
 export interface SQLiteDatabaseOptions {
   path: string
   timeoutMilliseconds?: number
   lockRetryAttempts?: number
+  synchronous?: SQLiteSynchronous
 }
 
 class SQLiteConnection implements DatabaseConnection {
@@ -57,10 +60,16 @@ export class SQLiteDatabase implements Database {
   private closed = false
   private readonly timeoutMilliseconds: number
   private readonly lockRetryAttempts: number
+  private readonly synchronous: SQLiteSynchronous
+  private journalConfigured = false
 
   constructor(options: SQLiteDatabaseOptions) {
     this.timeoutMilliseconds = options.timeoutMilliseconds ?? 5_000
     this.lockRetryAttempts = options.lockRetryAttempts ?? 10
+    this.synchronous = options.synchronous ?? "normal"
+    if (this.synchronous !== "normal" && this.synchronous !== "full") {
+      throw new TypeError('synchronous must be "normal" or "full"')
+    }
     if (!Number.isSafeInteger(this.lockRetryAttempts) || this.lockRetryAttempts < 1) {
       throw new TypeError("lockRetryAttempts must be a positive safe integer")
     }
@@ -115,12 +124,24 @@ export class SQLiteDatabase implements Database {
     }
     try {
       if (initialRemaining !== undefined) requireDatabaseDeadlineRemaining()
-      return await this.withBusyTimeout(callback)
+      return await this.withBusyTimeout(async () => {
+        await this.configureJournalWithRetry()
+        return callback()
+      })
     } finally {
       release()
     }
   }
 
+  private async configureJournalWithRetry(): Promise<void> {
+    if (this.journalConfigured) return
+    await this.retryWhileBusy(() => {
+      this.database.exec("PRAGMA journal_mode = WAL")
+      this.database.exec(`PRAGMA synchronous = ${this.synchronous.toUpperCase()}`)
+    })
+    this.journalConfigured = true
+  }
+
   private async withBusyTimeout<Result>(callback: () => Promise<Result>): Promise<Result> {
     const remaining = requireDatabaseDeadlineRemaining()
     if (remaining === undefined) return callback()
@@ -150,10 +171,14 @@ export class SQLiteDatabase implements Database {
   }
 
   private async beginTransactionWithRetry(): Promise<void> {
+    await this.retryWhileBusy(() => this.database.exec("BEGIN IMMEDIATE"))
+  }
+
+  private async retryWhileBusy(operation: () => void): Promise<void> {
     let retries = 0
     while (true) {
       try {
-        this.database.exec("BEGIN IMMEDIATE")
+        operation()
         return
       } catch (error) {
         if (databaseDeadlineRemainingMilliseconds() !== undefined) throw error

test/database-deadline.test.ts

@@ -93,6 +93,7 @@ describe("database deadlines", () => {
   it("turns SQLite lock contention into a deadline error", async () => {
     const path = `/tmp/solid-objects-deadline-${crypto.randomUUID()}.sqlite3`
     const blocker = new DatabaseSync(path)
+    blocker.exec("PRAGMA journal_mode = WAL")
     blocker.exec("CREATE TABLE records(id INTEGER PRIMARY KEY)")
     blocker.exec("BEGIN IMMEDIATE")
     const database = sqlite({ path, timeoutMilliseconds: 5_000 })

test/sqlite.test.ts

@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it } from "vitest"
 import { rm } from "node:fs/promises"
 import { DatabaseSync } from "node:sqlite"
 import { Worker } from "node:worker_threads"
-import { sqlite, type SQLiteDatabase } from "../src/database/sqlite.js"
+import { sqlite, type SQLiteDatabase, type SQLiteSynchronous } from "../src/database/sqlite.js"
 
 let database: SQLiteDatabase | undefined
 let blocker: Worker | undefined
@@ -18,9 +18,42 @@ afterEach(async () => {
 })
 
 describe("SQLite adapter", () => {
+  it("opens a file database with write-ahead logging and normal sync by default", async () => {
+    path = `/tmp/solid-objects-sqlite-${crypto.randomUUID()}.sqlite3`
+    database = sqlite({ path })
+
+    await expect(journalSettings(database)).resolves.toEqual({ journal: "wal", synchronous: 1 })
+  })
+
+  it("opens a file database with write-ahead logging and full sync when synchronous is full", async () => {
+    path = `/tmp/solid-objects-sqlite-${crypto.randomUUID()}.sqlite3`
+    database = sqlite({ path, synchronous: "full" })
+
+    await expect(journalSettings(database)).resolves.toEqual({ journal: "wal", synchronous: 2 })
+  })
+
+  it("keeps the memory journal for an in-memory database", async () => {
+    database = sqlite({ path: ":memory:" })
+
+    await expect(journalSettings(database)).resolves.toEqual({ journal: "memory", synchronous: 1 })
+  })
+
+  it("switches a rollback-journal file to write-ahead logging after another writer releases its lock", async () => {
+    path = `/tmp/solid-objects-sqlite-${crypto.randomUUID()}.sqlite3`
+    const setup = new DatabaseSync(path)
+    setup.exec("CREATE TABLE records(id INTEGER PRIMARY KEY)")
+    setup.close()
+    blocker = lockDatabase(path, { holdMilliseconds: 50 })
+    await nextMessage(blocker)
+    database = sqlite({ path, timeoutMilliseconds: 1, lockRetryAttempts: 10 })
+
+    await expect(journalSettings(database)).resolves.toEqual({ journal: "wal", synchronous: 1 })
+  })
+
   it("retries a transaction after transient writer contention", async () => {
     path = `/tmp/solid-objects-sqlite-${crypto.randomUUID()}.sqlite3`
     const setup = new DatabaseSync(path)
+    setup.exec("PRAGMA journal_mode = WAL")
     setup.exec("CREATE TABLE records(id INTEGER PRIMARY KEY)")
     setup.close()
     blocker = lockDatabase(path)
@@ -38,6 +71,12 @@ describe("SQLite adapter", () => {
     ).resolves.toEqual({ count: 1n })
   })
 
+  it("rejects a synchronous level other than normal or full", () => {
+    expect(() => sqlite({ path: ":memory:", synchronous: "off" as SQLiteSynchronous })).toThrow(
+      new TypeError('synchronous must be "normal" or "full"'),
+    )
+  })
+
   it("validates its transaction retry budget", () => {
     expect(() => sqlite({ path: ":memory:", lockRetryAttempts: 0 })).toThrow(
       "lockRetryAttempts must be a positive safe integer",
@@ -47,6 +86,7 @@ describe("SQLite adapter", () => {
   it("stops retrying after its transaction retry budget", async () => {
     path = `/tmp/solid-objects-sqlite-${crypto.randomUUID()}.sqlite3`
     const setup = new DatabaseSync(path)
+    setup.exec("PRAGMA journal_mode = WAL")
     setup.exec("CREATE TABLE records(id INTEGER PRIMARY KEY)")
     setup.close()
     blocker = lockDatabase(path, { holdMilliseconds: 1_000 })
@@ -64,6 +104,16 @@ describe("SQLite adapter", () => {
   })
 })
 
+async function journalSettings(
+  target: SQLiteDatabase,
+): Promise<{ journal: string; synchronous: number }> {
+  return target.connection(async (connection) => {
+    const journal = await connection.get<{ journal_mode: string }>("PRAGMA journal_mode")
+    const synchronous = await connection.get<{ synchronous: number | bigint }>("PRAGMA synchronous")
+    return { journal: journal!.journal_mode, synchronous: Number(synchronous!.synchronous) }
+  })
+}
+
 function lockDatabase(databasePath: string, options: { holdMilliseconds?: number } = {}): Worker {
   return new Worker(
     `