Add data directory locking and partial corrupt initdb cleanup

.changeset/add-nodefs-lock-file.md

@@ -0,0 +1,5 @@
+---
+'@electric-sql/pglite': patch
+---
+
+Add data directory locking and partial initdb recovery to NodeFS

.gitignore

@@ -16,3 +16,7 @@ node_modules
 
 docs/.vitepress/dist
 docs/.vitepress/cache
+
+# Local test logs
+packages/pglite/tests/crash-safety/*.log
+prompt_history.txt

packages/pglite/src/fs/nodefs.ts

@@ -4,8 +4,11 @@ import { EmscriptenBuiltinFilesystem, PGDATA } from './base.js'
 import type { PostgresMod } from '../postgresMod.js'
 import { PGlite } from '../pglite.js'
 
+// TODO: Add locking for browser backends via Web Locks API
+
 export class NodeFS extends EmscriptenBuiltinFilesystem {
   protected rootDir: string
+  #lockFd: number | null = null
 
   constructor(dataDir: string) {
     super(dataDir)
@@ -17,6 +20,10 @@ export class NodeFS extends EmscriptenBuiltinFilesystem {
 
   async init(pg: PGlite, opts: Partial<PostgresMod>) {
     this.pg = pg
+
+    this.#acquireLock()
+    this.#cleanPartialInit()
+
     const options: Partial<PostgresMod> = {
       ...opts,
       preRun: [
@@ -31,7 +38,105 @@ export class NodeFS extends EmscriptenBuiltinFilesystem {
     return { emscriptenOpts: options }
   }
 
+  // Lock file is a sibling (mydb.lock) to avoid polluting the PG data dir
+  #acquireLock() {
+    const lockPath = this.rootDir + '.lock'
+
+    if (fs.existsSync(lockPath)) {
+      try {
+        const content = fs.readFileSync(lockPath, 'utf-8').trim()
+        const lines = content.split('\n')
+        const pid = parseInt(lines[0], 10)
+
+        if (pid && !isNaN(pid) && this.#isProcessAlive(pid)) {
+          throw new Error(
+            `Data directory "${this.rootDir}" is locked by another PGlite instance ` +
+              `(PID ${pid}). Close the other instance first, or delete ` +
+              `"${lockPath}" if the process is no longer running.`,
+          )
+        }
+        // Stale lock from a dead process — safe to take over
+      } catch (e) {
+        // Re-throw lock errors, ignore parse errors (corrupt lock file = stale)
+        if (e instanceof Error && e.message.includes('is locked by')) {
+          throw e
+        }
+      }
+    }
+
+    // Write our PID to the lock file and keep the fd open
+    this.#lockFd = fs.openSync(lockPath, 'w')
+    fs.writeSync(this.#lockFd, `${process.pid}\n${Date.now()}\n`)
+  }
+
+  #releaseLock() {
+    if (this.#lockFd !== null) {
+      try {
+        fs.closeSync(this.#lockFd)
+      } catch {
+        // Ignore errors on close
+      }
+      this.#lockFd = null
+
+      const lockPath = this.rootDir + '.lock'
+      try {
+        fs.unlinkSync(lockPath)
+      } catch {
+        // Ignore errors on unlink (dir may already be cleaned up)
+      }
+    }
+  }
+
+  // If initdb was killed mid-way, the data dir is incomplete and unrecoverable.
+  // A fully initialized PG always has 3+ databases in base/ (template0, template1, postgres).
+  #cleanPartialInit() {
+    try {
+      const entries = fs.readdirSync(this.rootDir)
+      if (entries.length === 0) return
+
+      const pgVersionPath = path.join(this.rootDir, 'PG_VERSION')
+      if (!fs.existsSync(pgVersionPath)) {
+        this.#moveDataDirToBackup()
+        return
+      }
+
+      const basePath = path.join(this.rootDir, 'base')
+      if (fs.existsSync(basePath)) {
+        const databases = fs.readdirSync(basePath)
+        if (databases.length < 3) {
+          this.#moveDataDirToBackup()
+          return
+        }
+      } else {
+        this.#moveDataDirToBackup()
+      }
+    } catch {
+      // If we can't read the directory, let PostgreSQL handle the error
+    }
+  }
+
+  #moveDataDirToBackup() {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
+    const backupPath = `${this.rootDir}.corrupt-${timestamp}`
+    fs.renameSync(this.rootDir, backupPath)
+    fs.mkdirSync(this.rootDir)
+    console.warn(
+      `PGlite: Detected partially-initialized data directory. ` +
+        `Moved to "${backupPath}" for recovery. A fresh database will be created.`,
+    )
+  }
+
+  #isProcessAlive(pid: number): boolean {
+    try {
+      process.kill(pid, 0) // signal 0 = check if process exists
+      return true
+    } catch {
+      return false // ESRCH = process doesn't exist
+    }
+  }
+
   async closeFs(): Promise<void> {
+    this.#releaseLock()
     this.pg!.Module.FS.quit()
   }
 }

packages/pglite/tests/crash-safety/README.md

@@ -0,0 +1,133 @@
+# PGlite Crash Safety Test Suite
+
+Tests that reproduce real corruption bugs in PGlite — specifically the issues fixed by the PID file lock and partial initdb detection in `nodefs.ts`. Every test here **fails without the fix** and passes with it.
+
+Single-instance WAL recovery tests (kill-during-insert, kill-during-transaction, etc.) were intentionally excluded because PostgreSQL handles those correctly without any code changes. Those tests are preserved in the `archive/all-crash-safety-tests` branch.
+
+## Running the Tests
+
+```bash
+# Run all crash safety tests
+pnpm vitest run tests/crash-safety/ --reporter=verbose
+
+# Run a single scenario
+pnpm vitest run tests/crash-safety/overlapping-instances.test.js
+
+# Keep data directories for debugging (not cleaned up after test)
+RETAIN_DATA=1 pnpm vitest run tests/crash-safety/
+```
+
+> **Note:** Do not use `--no-file-parallelism` — PGlite's WASM module conflicts with vitest's single-worker mode.
+
+## Architecture
+
+```
+tests/crash-safety/
+├── harness.js                          # Shared test infrastructure
+├── README.md                           # This file
+├── CRASH-SAFETY.md                     # Detailed failure mode documentation
+├── RESULTS.md                          # Test results log
+├── hmr-double-instance.test.js         # HMR double-instance lock test
+├── overlapping-instances.test.js       # Overlapping instance corruption test
+├── wal-bloat-no-checkpoint.test.js     # WAL bloat burst mode corruption test
+├── partial-init-backup.test.js         # Partial initdb backup behavior test
+└── workers/
+    ├── hmr-double-instance.js
+    ├── overlapping-three-instances.js
+    ├── overlapping-staggered.js
+    ├── overlapping-ddl-writer.js
+    ├── overlapping-rapid-cycling.js
+    └── wal-bloat-no-checkpoint.js
+```
+
+### How It Works
+
+Each test follows the same pattern:
+
+1. **Worker script** — A standalone Node.js script that creates a PGlite instance on a data directory (passed via `PGLITE_DATA_DIR` env var), performs database operations, and sends IPC messages to the parent via `process.send()` to signal progress.
+
+2. **Test file** — Uses vitest. Calls `crashTest()` from the harness to spawn the worker as a child process via `fork()`. The harness kills the child with `SIGKILL` either after a timer or when a specific IPC message is received.
+
+3. **Verification** — After the kill, the test reopens PGlite on the same data directory and checks:
+
+   - The database opens without error (no PANIC, no hang)
+   - Basic queries succeed (`SELECT 1`)
+   - All user tables are scannable
+   - Data is consistent (committed rows present, uncommitted rows absent)
+
+4. **Cleanup** — Each test uses a unique `/tmp/pglite-crash-*` directory and removes it in `afterAll`, unless `RETAIN_DATA=1` is set.
+
+## Test Scenarios
+
+### 1. Overlapping Instances
+
+**File:** `overlapping-instances.test.js` (4 tests)
+
+Multiple PGlite instances opening the same data directory concurrently. Without the PID file lock, this causes silent corruption (`Aborted()` on next open).
+
+- **Triple instances** — three instances open simultaneously
+- **Staggered** — second instance opens while first is mid-write
+- **DDL writer** — overlapping DDL operations
+- **Rapid cycling** — rapid open/kill/reopen cycles with overlapping lifetimes
+
+### 2. HMR Double-Instance
+
+**File:** `hmr-double-instance.test.js` (2 tests)
+
+Simulates hot module replacement (HMR) in dev servers where a new PGlite instance is created before the old one is closed.
+
+- **Lock blocking** — verifies instance B is blocked by the lock while instance A is alive
+- **Rapid HMR cycles** — fast instance swaps that corrupt without the lock
+
+### 3. WAL Bloat Burst Mode
+
+**File:** `wal-bloat-no-checkpoint.test.js` (1 failing test)
+
+15 extremely rapid kill cycles with no delay, accumulating WAL without checkpointing. Without partial initdb detection, interrupted initializations leave corrupt state that causes `Aborted()`.
+
+### 4. Partial Init Backup
+
+**File:** `partial-init-backup.test.js` (3 tests)
+
+Directly tests the partial initdb detection and backup behavior in `nodefs.ts`:
+
+- Partial dir (no `PG_VERSION`) → moved to `.corrupt-<timestamp>` backup
+- Partial dir (`PG_VERSION` but incomplete `base/`) → moved to backup
+- Fully initialized dir → NOT moved (no false positives)
+
+## Harness API (`harness.js`)
+
+### `crashTest(options)`
+
+Spawns a child process and kills it.
+
+| Option          | Type   | Default     | Description                                                    |
+| --------------- | ------ | ----------- | -------------------------------------------------------------- |
+| `dataDir`       | string | required    | Path to PGlite data directory                                  |
+| `workerScript`  | string | required    | Path to the worker `.js` file                                  |
+| `killAfterMs`   | number | `500`       | Delay before sending kill signal                               |
+| `signal`        | string | `'SIGKILL'` | Signal to send (usually SIGKILL)                               |
+| `killOnMessage` | string | `null`      | Kill when worker sends this IPC message instead of using timer |
+| `env`           | object | `{}`        | Extra environment variables for the child                      |
+
+Returns: `{ workerKilled, workerError, workerMessages, workerExitCode, workerSignal, stdout, stderr }`
+
+### `tryOpen(dataDir, timeoutMs?)`
+
+Attempts to open a PGlite instance on a potentially corrupted data directory. Includes a timeout (default 15s) to handle cases where a corrupted database hangs forever during initialization.
+
+Returns: `{ success, db, error }`
+
+### `verifyIntegrity(db)`
+
+Runs integrity checks against an open PGlite instance: basic query, table scan, index scan.
+
+Returns: `{ intact, issues }`
+
+### `cleanupDataDir(dataDir)`
+
+Removes a test data directory and its sibling `.lock` file.
+
+### `testDataDir(scenarioName)`
+
+Generates a unique `/tmp/pglite-crash-<name>-<timestamp>-<rand>` path.