diff --git a/README.md b/README.md index ae19fde..0bdbdc6 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,13 @@ fingerprints, summaries, durable clusters, and PR detail/file rows from that same image, negotiates the remote snapshot-provenance contract before touching R2, uploads its digest-scoped bundle, and completes staged D1 coverage through row- and encoded-byte-bounded ingest requests. +The cloud SQLite image preserves full canonical issue and pull-request bodies, +comments, revisions, review comments, and PR patch text. Gzip is lossless +transport, not portable-store compaction. Bundle privacy metadata declares both +message bodies and source code because patch text is retained. Publication still +removes raw API payloads, blob-backed payloads, local run diagnostics, source +code indexes, and machine-local paths; those are not part of the shared archive +contract. Publishing moves unpinned reads to the complete snapshot by default, preserving the existing reader-refresh behavior. The remote must advertise `gitcrawl.snapshot.staging.v1`; `--stage-only` keeps the immutable snapshot diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index deadf87..67c1ff7 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -472,16 +472,18 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { `).Scan(&repositoryRawJSON, &threadRawJSON, &detailRawJSON, &filePatch, &fileRawJSON); err != nil { t.Fatalf("inspect scrubbed cloud payloads: %v", err) } - if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || filePatch != "" || fileRawJSON != "" { + if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || fileRawJSON != "" { t.Fatalf( - "cloud snapshot retained private payloads: repository=%q thread=%q detail=%q patch=%q file=%q", + "cloud snapshot retained raw payloads: repository=%q thread=%q detail=%q file=%q", repositoryRawJSON, threadRawJSON, detailRawJSON, - filePatch, fileRawJSON, ) } + if filePatch != "@@ private source" { + t.Fatalf("cloud snapshot patch = %q, want full structured patch text", filePatch) + } var blobCount int if err := snapshotDB.QueryRowContext(ctx, `select count(*) from blobs`).Scan(&blobCount); err != nil { t.Fatalf("inspect cloud blobs: %v", err) @@ -1200,6 +1202,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "bundle privacy should disclose message bodies", http.StatusBadRequest) return } + if manifest.Privacy["includes_source_code"] != true { + http.Error(w, "bundle privacy should disclose patch text", http.StatusBadRequest) + return + } if manifest.SnapshotID != snapshotID || manifest.Object.Key != crawlremote.SQLiteSnapshotObjectKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID) { http.Error(w, "bundle snapshot mismatch", http.StatusBadRequest) @@ -1413,7 +1419,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } } privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any) - if !ok || privacy["includes_private_messages"] != true { + if !ok || + privacy["includes_private_messages"] != true || + privacy["includes_source_code"] != true { t.Fatalf("missing sqlite bundle privacy output: %#v", payload) } } diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 3c64475..0736ec3 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -1664,7 +1664,7 @@ func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, "includes_raw_json": false, - "includes_source_code": false, + "includes_source_code": true, } } @@ -1758,7 +1758,6 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { }{ {table: "comments", name: "raw_json_blob_id"}, {table: "thread_revisions", name: "raw_json_blob_id"}, - {table: "pull_request_files", name: "patch"}, } { exists, err := sqliteColumnExists(ctx, db, column.table, column.name) if err != nil { @@ -1767,13 +1766,9 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { if !exists { continue } - value := "null" - if column.name == "patch" { - value = "''" - } if _, err := db.ExecContext( ctx, - `update `+column.table+` set `+column.name+` = `+value+` where `+column.name+` is not null`, + `update `+column.table+` set `+column.name+` = null where `+column.name+` is not null`, ); err != nil { return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err) } diff --git a/internal/store/comments.go b/internal/store/comments.go index 0f9bdf2..f2c599e 100644 --- a/internal/store/comments.go +++ b/internal/store/comments.go @@ -71,6 +71,16 @@ func (s *Store) upsertComment(ctx context.Context, comment Comment) (int64, erro if err != nil { return 0, fmt.Errorf("upsert comment: %w", err) } + if s.portableCommentBodyMetadata { + if _, err := s.q().ExecContext(ctx, ` + update comments + set body_length = length(body), + body_excerpt = substr(body, 1, ?) + where id = ? + `, s.portableBodyChars, id); err != nil { + return 0, fmt.Errorf("refresh portable comment body metadata: %w", err) + } + } if err := s.recordCommentRevision(ctx, id, time.Now().UTC().Format(timeLayout)); err != nil { return 0, err } diff --git a/internal/store/store.go b/internal/store/store.go index f3ece75..44be78b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -34,10 +34,13 @@ type sqliteCoder interface { } type Store struct { - db *sql.DB - queries dbQueries - sqlc *storedb.Queries - path string + db *sql.DB + queries dbQueries + sqlc *storedb.Queries + path string + portableThreadBodyMetadata bool + portableCommentBodyMetadata bool + portableBodyChars int } type dbQueries interface { @@ -67,6 +70,7 @@ func Open(ctx context.Context, path string) (*Store, error) { _ = base.Close() return nil, err } + st.detectPortableBodyMetadata(ctx) return st, nil } @@ -166,7 +170,15 @@ func (s *Store) withTxOnce(ctx context.Context, fn func(*Store) error) error { if err != nil { return fmt.Errorf("begin transaction: %w", err) } - txStore := &Store{db: s.db, queries: tx, sqlc: s.qsql().WithTx(tx), path: s.path} + txStore := &Store{ + db: s.db, + queries: tx, + sqlc: s.qsql().WithTx(tx), + path: s.path, + portableThreadBodyMetadata: s.portableThreadBodyMetadata, + portableCommentBodyMetadata: s.portableCommentBodyMetadata, + portableBodyChars: s.portableBodyChars, + } if err := fn(txStore); err != nil { _ = tx.Rollback() return err @@ -854,6 +866,26 @@ func (s *Store) hasColumn(ctx context.Context, table, column string) bool { return false } +func (s *Store) detectPortableBodyMetadata(ctx context.Context) { + s.portableThreadBodyMetadata = s.hasColumns(ctx, "threads", "body_excerpt", "body_length") + s.portableCommentBodyMetadata = s.hasColumns(ctx, "comments", "body_excerpt", "body_length") + if !s.portableThreadBodyMetadata && !s.portableCommentBodyMetadata { + return + } + s.portableBodyChars = 256 + if !s.hasColumns(ctx, "portable_metadata", "key", "value") { + return + } + var bodyChars int + if err := s.q().QueryRowContext(ctx, ` + select cast(value as integer) + from portable_metadata + where key = 'body_chars' + `).Scan(&bodyChars); err == nil && bodyChars > 0 { + s.portableBodyChars = bodyChars + } +} + func (s *Store) schemaVersion(ctx context.Context) (int, error) { var version int if err := s.db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5269f4d..486f1f5 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2251,3 +2251,131 @@ func TestPrunePortablePayloads(t *testing.T) { t.Fatalf("payloads not pruned: bodyExcerpt=%q titleTokens=%q linkedRefs=%q buckets=%q features=%q", bodyExcerpt, titleTokens, linkedRefs, buckets, features) } } + +func TestHydrationRefreshesPortableBodyMetadata(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "gitcrawl.db") + st, err := Open(ctx, dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + + repoID, err := st.UpsertRepository(ctx, Repository{ + Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", + RawJSON: "{}", UpdatedAt: "2026-07-29T00:00:00Z", + }) + if err != nil { + t.Fatalf("repo: %v", err) + } + threadID, err := st.UpsertThread(ctx, Thread{ + RepoID: repoID, GitHubID: "1", Number: 1, Kind: "issue", State: "open", + Title: "full cloud archive", Body: "old", + HTMLURL: "https://github.com/openclaw/gitcrawl/issues/1", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"old"}`, + ContentHash: "original", UpdatedAtGitHub: "2026-07-29T00:00:00Z", + UpdatedAt: "2026-07-29T00:00:00Z", + }) + if err != nil { + t.Fatalf("thread: %v", err) + } + if _, err := st.UpsertComment(ctx, Comment{ + ThreadID: threadID, GitHubID: "c1", CommentType: "issue_comment", + Body: "old", RawJSON: "{}", + CreatedAtGitHub: "2026-07-29T00:00:00Z", + }); err != nil { + t.Fatalf("comment: %v", err) + } + if _, err := st.PrunePortablePayloads(ctx, PortablePruneOptions{BodyChars: 8}); err != nil { + t.Fatalf("prune: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close portable store: %v", err) + } + + st, err = Open(ctx, dbPath) + if err != nil { + t.Fatalf("reopen portable store: %v", err) + } + defer st.Close() + + if _, err := st.UpsertThread(ctx, Thread{ + RepoID: repoID, GitHubID: "1", Number: 1, Kind: "issue", State: "open", + Title: "full cloud archive", Body: "new body is longer", + HTMLURL: "https://github.com/openclaw/gitcrawl/issues/1", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"new body is longer"}`, + ContentHash: "hydrated", UpdatedAtGitHub: "2026-07-29T01:00:00Z", + UpdatedAt: "2026-07-29T01:00:00Z", + }); err != nil { + t.Fatalf("hydrate thread: %v", err) + } + if _, err := st.UpsertComment(ctx, Comment{ + ThreadID: threadID, GitHubID: "c1", CommentType: "issue_comment", + Body: "new comment is longer", RawJSON: "{}", CreatedAtGitHub: "2026-07-29T00:00:00Z", + UpdatedAtGitHub: "2026-07-29T01:00:00Z", + }); err != nil { + t.Fatalf("hydrate comment: %v", err) + } + insertedThreadID, err := st.UpsertThread(ctx, Thread{ + RepoID: repoID, GitHubID: "2", Number: 2, Kind: "issue", State: "open", + Title: "new cloud row", Body: "inserted body is longer", + HTMLURL: "https://github.com/openclaw/gitcrawl/issues/2", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"inserted body is longer"}`, + ContentHash: "inserted", UpdatedAtGitHub: "2026-07-29T01:00:00Z", + UpdatedAt: "2026-07-29T01:00:00Z", + }) + if err != nil { + t.Fatalf("insert hydrated thread: %v", err) + } + if _, err := st.UpsertComment(ctx, Comment{ + ThreadID: insertedThreadID, GitHubID: "c2", CommentType: "issue_comment", + Body: "inserted comment is longer", RawJSON: "{}", + CreatedAtGitHub: "2026-07-29T01:00:00Z", + }); err != nil { + t.Fatalf("insert hydrated comment: %v", err) + } + + var threadBody, threadExcerpt string + var threadBodyLength int + if err := st.DB().QueryRowContext(ctx, ` + select body, body_excerpt, body_length from threads where id = ? + `, threadID).Scan(&threadBody, &threadExcerpt, &threadBodyLength); err != nil { + t.Fatalf("read hydrated thread: %v", err) + } + if threadBody != "new body is longer" || + threadExcerpt != "new body" || + threadBodyLength != len("new body is longer") { + t.Fatalf("hydrated thread metadata = body %q excerpt %q length %d", threadBody, threadExcerpt, threadBodyLength) + } + + var commentBody, commentExcerpt string + var commentBodyLength int + if err := st.DB().QueryRowContext(ctx, ` + select body, body_excerpt, body_length from comments where github_id = 'c1' + `).Scan(&commentBody, &commentExcerpt, &commentBodyLength); err != nil { + t.Fatalf("read hydrated comment: %v", err) + } + if commentBody != "new comment is longer" || + commentExcerpt != "new comm" || + commentBodyLength != len("new comment is longer") { + t.Fatalf("hydrated comment metadata = body %q excerpt %q length %d", commentBody, commentExcerpt, commentBodyLength) + } + + var insertedThreadExcerpt, insertedCommentExcerpt string + if err := st.DB().QueryRowContext(ctx, ` + select body_excerpt from threads where id = ? + `, insertedThreadID).Scan(&insertedThreadExcerpt); err != nil { + t.Fatalf("read inserted thread excerpt: %v", err) + } + if err := st.DB().QueryRowContext(ctx, ` + select body_excerpt from comments where github_id = 'c2' + `).Scan(&insertedCommentExcerpt); err != nil { + t.Fatalf("read inserted comment excerpt: %v", err) + } + if insertedThreadExcerpt != "inserted" || insertedCommentExcerpt != "inserted" { + t.Fatalf( + "inserted excerpts = thread %q comment %q", + insertedThreadExcerpt, + insertedCommentExcerpt, + ) + } +} diff --git a/internal/store/threads.go b/internal/store/threads.go index 4b8675c..0a3dd74 100644 --- a/internal/store/threads.go +++ b/internal/store/threads.go @@ -254,6 +254,19 @@ func (s *Store) upsertThreadObservation(ctx context.Context, thread Thread, opti if err != nil { return UpsertThreadResult{}, fmt.Errorf("upsert thread: %w", err) } + if s.portableThreadBodyMetadata { + if _, err := s.q().ExecContext(ctx, ` + update threads + set body_length = length(coalesce(body, '')), + body_excerpt = case + when body is null then '' + else substr(body, 1, ?) + end + where id = ? + `, s.portableBodyChars, id); err != nil { + return UpsertThreadResult{}, fmt.Errorf("refresh portable thread body metadata: %w", err) + } + } evidenceApplied := false if !options.IncompleteEvidence { evidenceApplied, err = s.reserveThreadEvidenceObservation(