core: open the cached catalog as a file rather than copying it into memory

Now that analysis only reads the catalog, a restored one no longer has to
be a private in-memory copy. The stored blob is opened where it lies: it is
named after the hash of its contents, so it can never change under a
reader, and SQLite is told as much with immutable=1. Any number of
processes share the one file, with no locking and no copy — eight parallel
runs against one blob check out, and a write to it is refused.

CAS grows a Filename accessor for consumers that need the file rather than
its bytes. A catalog being built is still an in-memory database, since
building one is all writes; only the finished blob is a file.

PostgreSQL analyze goes from 21ms to 19ms per warm run and stops copying
720KB per run to do it.

Co-Authored-By: Claude <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011MnoUabwBWW9gaEn2Nj7eG

Author: claude

internal/cache/cas.go

@@ -41,6 +41,19 @@ func (c *CAS) path(d Digest) string {
 	return filepath.Join("cas", d.Hash[:2], d.Hash)
 }
 
+// Filename returns the path of a stored blob, for a consumer that needs the
+// file rather than its bytes — SQLite, for one, opens a database by name. A
+// blob is named after the hash of its contents, so the file at this path never
+// changes and any number of processes may read it at once.
+//
+// It reports false when the blob is not stored.
+func (c *CAS) Filename(d Digest) (string, bool) {
+	if !c.Contains(d) {
+		return "", false
+	}
+	return filepath.Join(c.root.Name(), c.path(d)), true
+}
+
 // createTemp creates a staging file under tmp/ in the cache root, returning
 // the open file and its root-relative name.
 func (c *CAS) createTemp(prefix string) (*os.File, string, error) {

internal/core/cache.go

@@ -18,7 +18,6 @@ const catalogOutput = "catalog.db"
 // implements on its connections.
 type serializer interface {
 	Serialize() ([]byte, error)
-	Deserialize([]byte) error
 }
 
 // NewCached returns the catalog a dialect and a schema produce, restored from
@@ -27,8 +26,8 @@ type serializer interface {
 // Producing one means seeding the dialect — thousands of rows, since
 // PostgreSQL's functions and system catalogs alone run to five figures — and
 // then parsing the schema and applying its DDL. The result is the same every
-// time, so the finished database is serialized into the cache and
-// deserialized back on the next run, turning all of that work into one read.
+// time, so the finished database is stored in the cache and opened directly on
+// the next run, turning all of that work into opening a file.
 // It is a single action: the dialect and the schema together are what the
 // catalog is, so they are one key, and a hit means apply is never called.
 //
@@ -81,25 +80,23 @@ func openAction(dialect string, schema []string) (*cache.Cache, cache.Digest) {
 	return store, action.Digest()
 }
 
-// restore opens the catalog a previous run cached.
+// restore opens the catalog a previous run cached. The stored blob is opened
+// where it lies rather than copied: analysis only ever reads the catalog, so
+// every run that wants this one reads the same file.
 func restore(store *cache.Cache, action cache.Digest) (*Catalog, error) {
 	result, err := store.Actions.Get(action)
 	if err != nil {
 		return nil, err
 	}
-	blob, err := store.CAS.Get(result.Outputs[catalogOutput])
-	if err != nil {
-		return nil, err
+	path, ok := store.CAS.Filename(result.Outputs[catalogOutput])
+	if !ok {
+		return nil, cache.ErrNotFound
 	}
 
-	db, err := openDB()
+	db, err := openFile(path)
 	if err != nil {
 		return nil, err
 	}
-	if err := deserialize(db, blob); err != nil {
-		db.Close()
-		return nil, err
-	}
 
 	stmts := newStmtCache(db)
 	cat := &Catalog{db: db, stmts: stmts, q: catalogdb.New(stmts)}
@@ -130,7 +127,8 @@ func save(store *cache.Cache, action cache.Digest, cat *Catalog) error {
 	})
 }
 
-// serialize returns the bytes a database would be written to disk as.
+// serialize returns the bytes a database being built in memory would be
+// written to disk as, which is what gets stored.
 func serialize(db *sql.DB) ([]byte, error) {
 	var blob []byte
 	err := withRawConn(db, func(s serializer) error {
@@ -144,18 +142,6 @@ func serialize(db *sql.DB) ([]byte, error) {
 	return blob, nil
 }
 
-// deserialize loads a serialized database into an open one. The pool is
-// pinned to a single connection, so the connection this replaces the contents
-// of is the only one the catalog will ever use.
-func deserialize(db *sql.DB, blob []byte) error {
-	if err := withRawConn(db, func(s serializer) error {
-		return s.Deserialize(blob)
-	}); err != nil {
-		return fmt.Errorf("core: deserialize catalog: %w", err)
-	}
-	return nil
-}
-
 func withRawConn(db *sql.DB, fn func(serializer) error) error {
 	conn, err := db.Conn(context.Background())
 	if err != nil {

internal/core/catalog.go

@@ -64,23 +64,40 @@ func New(opts ...Option) (*Catalog, error) {
 	return c, nil
 }
 
-// openDB opens the empty SQLite database a catalog is kept in.
-func openDB() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", ":memory:")
+// openFile opens a catalog held in a file, read only. The file is named after
+// the hash of its contents, so it can never change under a reader: SQLite is
+// told as much, which lets any number of processes share it with no locking
+// and no copy.
+func openFile(path string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&immutable=1")
 	if err != nil {
-		return nil, fmt.Errorf("core: open catalog: %w", err)
+		return nil, fmt.Errorf("core: open catalog %s: %w", path, err)
 	}
-	// Every ":memory:" connection is its own empty database, so the pool has to
-	// be pinned to a single connection that is never retired — otherwise a
-	// second connection would see a catalog with no tables in it.
+	pinPool(db)
+	return db, nil
+}
+
+// pinPool holds the catalog to a single connection that is never retired.
+// Every ":memory:" connection is its own empty database, so a second one would
+// see a catalog with no tables in it; a catalog read from a file has no such
+// constraint, but has no use for more connections either.
+func pinPool(db *sql.DB) {
 	db.SetMaxOpenConns(1)
 	db.SetMaxIdleConns(1)
 	db.SetConnMaxIdleTime(0)
 	db.SetConnMaxLifetime(0)
+}
+
+// openDB opens the empty SQLite database a catalog is built in.
+func openDB() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		return nil, fmt.Errorf("core: open catalog: %w", err)
+	}
+	pinPool(db)
 
-	// The catalog is scratch state rebuilt on every run and never read back
-	// from disk, so durability buys nothing and costs a journal write per
-	// statement.
+	// The catalog being built is scratch state that is never read back from
+	// disk, so durability buys nothing and costs a journal write per statement.
 	for _, pragma := range []string{
 		"PRAGMA journal_mode = OFF",
 		"PRAGMA synchronous = OFF",