fix(cloud): preserve full PR patch fidelity

README.md

@@ -73,6 +73,13 @@ fingerprints, summaries, durable clusters, and PR detail/file rows from that
 same image, negotiates the remote snapshot-provenance contract before touching
 R2, uploads its digest-scoped bundle, and completes staged D1 coverage through
 row- and encoded-byte-bounded ingest requests.
+The cloud SQLite image preserves full canonical issue and pull-request bodies,
+comments, revisions, review comments, and PR patch text. Gzip is lossless
+transport, not portable-store compaction. Bundle privacy metadata declares both
+message bodies and source code because patch text is retained. Publication still
+removes raw API payloads, blob-backed payloads, local run diagnostics, source
+code indexes, and machine-local paths; those are not part of the shared archive
+contract.
 Publishing moves unpinned reads to the complete snapshot by default, preserving
 the existing reader-refresh behavior. The remote must advertise
 `gitcrawl.snapshot.staging.v1`; `--stage-only` keeps the immutable snapshot

internal/cli/app_test.go

@@ -472,16 +472,18 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) {
 	`).Scan(&repositoryRawJSON, &threadRawJSON, &detailRawJSON, &filePatch, &fileRawJSON); err != nil {
 		t.Fatalf("inspect scrubbed cloud payloads: %v", err)
 	}
-	if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || filePatch != "" || fileRawJSON != "" {
+	if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || fileRawJSON != "" {
 		t.Fatalf(
-			"cloud snapshot retained private payloads: repository=%q thread=%q detail=%q patch=%q file=%q",
+			"cloud snapshot retained raw payloads: repository=%q thread=%q detail=%q file=%q",
 			repositoryRawJSON,
 			threadRawJSON,
 			detailRawJSON,
-			filePatch,
 			fileRawJSON,
 		)
 	}
+	if filePatch != "@@ private source" {
+		t.Fatalf("cloud snapshot patch = %q, want full structured patch text", filePatch)
+	}
 	var blobCount int
 	if err := snapshotDB.QueryRowContext(ctx, `select count(*) from blobs`).Scan(&blobCount); err != nil {
 		t.Fatalf("inspect cloud blobs: %v", err)
@@ -1200,6 +1202,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) {
 					http.Error(w, "bundle privacy should disclose message bodies", http.StatusBadRequest)
 					return
 				}
+				if manifest.Privacy["includes_source_code"] != true {
+					http.Error(w, "bundle privacy should disclose patch text", http.StatusBadRequest)
+					return
+				}
 				if manifest.SnapshotID != snapshotID ||
 					manifest.Object.Key != crawlremote.SQLiteSnapshotObjectKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID) {
 					http.Error(w, "bundle snapshot mismatch", http.StatusBadRequest)
@@ -1413,7 +1419,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) {
 		}
 	}
 	privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any)
-	if !ok || privacy["includes_private_messages"] != true {
+	if !ok ||
+		privacy["includes_private_messages"] != true ||
+		privacy["includes_source_code"] != true {
 		t.Fatalf("missing sqlite bundle privacy output: %#v", payload)
 	}
 }

internal/cli/cloud_commands.go

@@ -1664,7 +1664,7 @@ func gitcrawlCloudSQLiteBundlePrivacy() map[string]any {
 	return map[string]any{
 		"includes_private_messages": true,
 		"includes_raw_json":         false,
-		"includes_source_code":      false,
+		"includes_source_code":      true,
 	}
 }
 
@@ -1758,7 +1758,6 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error {
 	}{
 		{table: "comments", name: "raw_json_blob_id"},
 		{table: "thread_revisions", name: "raw_json_blob_id"},
-		{table: "pull_request_files", name: "patch"},
 	} {
 		exists, err := sqliteColumnExists(ctx, db, column.table, column.name)
 		if err != nil {
@@ -1767,13 +1766,9 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error {
 		if !exists {
 			continue
 		}
-		value := "null"
-		if column.name == "patch" {
-			value = "''"
-		}
 		if _, err := db.ExecContext(
 			ctx,
-			`update `+column.table+` set `+column.name+` = `+value+` where `+column.name+` is not null`,
+			`update `+column.table+` set `+column.name+` = null where `+column.name+` is not null`,
 		); err != nil {
 			return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err)
 		}

internal/store/comments.go

@@ -71,6 +71,16 @@ func (s *Store) upsertComment(ctx context.Context, comment Comment) (int64, erro
 	if err != nil {
 		return 0, fmt.Errorf("upsert comment: %w", err)
 	}
+	if s.portableCommentBodyMetadata {
+		if _, err := s.q().ExecContext(ctx, `
+			update comments
+			set body_length = length(body),
+				body_excerpt = substr(body, 1, ?)
+			where id = ?
+		`, s.portableBodyChars, id); err != nil {
+			return 0, fmt.Errorf("refresh portable comment body metadata: %w", err)
+		}
+	}
 	if err := s.recordCommentRevision(ctx, id, time.Now().UTC().Format(timeLayout)); err != nil {
 		return 0, err
 	}

internal/store/store.go

@@ -34,10 +34,13 @@ type sqliteCoder interface {
 }
 
 type Store struct {
-	db      *sql.DB
-	queries dbQueries
-	sqlc    *storedb.Queries
-	path    string
+	db                          *sql.DB
+	queries                     dbQueries
+	sqlc                        *storedb.Queries
+	path                        string
+	portableThreadBodyMetadata  bool
+	portableCommentBodyMetadata bool
+	portableBodyChars           int
 }
 
 type dbQueries interface {
@@ -67,6 +70,7 @@ func Open(ctx context.Context, path string) (*Store, error) {
 		_ = base.Close()
 		return nil, err
 	}
+	st.detectPortableBodyMetadata(ctx)
 	return st, nil
 }
 
@@ -166,7 +170,15 @@ func (s *Store) withTxOnce(ctx context.Context, fn func(*Store) error) error {
 	if err != nil {
 		return fmt.Errorf("begin transaction: %w", err)
 	}
-	txStore := &Store{db: s.db, queries: tx, sqlc: s.qsql().WithTx(tx), path: s.path}
+	txStore := &Store{
+		db:                          s.db,
+		queries:                     tx,
+		sqlc:                        s.qsql().WithTx(tx),
+		path:                        s.path,
+		portableThreadBodyMetadata:  s.portableThreadBodyMetadata,
+		portableCommentBodyMetadata: s.portableCommentBodyMetadata,
+		portableBodyChars:           s.portableBodyChars,
+	}
 	if err := fn(txStore); err != nil {
 		_ = tx.Rollback()
 		return err
@@ -854,6 +866,26 @@ func (s *Store) hasColumn(ctx context.Context, table, column string) bool {
 	return false
 }
 
+func (s *Store) detectPortableBodyMetadata(ctx context.Context) {
+	s.portableThreadBodyMetadata = s.hasColumns(ctx, "threads", "body_excerpt", "body_length")
+	s.portableCommentBodyMetadata = s.hasColumns(ctx, "comments", "body_excerpt", "body_length")
+	if !s.portableThreadBodyMetadata && !s.portableCommentBodyMetadata {
+		return
+	}
+	s.portableBodyChars = 256
+	if !s.hasColumns(ctx, "portable_metadata", "key", "value") {
+		return
+	}
+	var bodyChars int
+	if err := s.q().QueryRowContext(ctx, `
+		select cast(value as integer)
+		from portable_metadata
+		where key = 'body_chars'
+	`).Scan(&bodyChars); err == nil && bodyChars > 0 {
+		s.portableBodyChars = bodyChars
+	}
+}
+
 func (s *Store) schemaVersion(ctx context.Context) (int, error) {
 	var version int
 	if err := s.db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil {

internal/store/store_test.go

@@ -2251,3 +2251,131 @@ func TestPrunePortablePayloads(t *testing.T) {
 		t.Fatalf("payloads not pruned: bodyExcerpt=%q titleTokens=%q linkedRefs=%q buckets=%q features=%q", bodyExcerpt, titleTokens, linkedRefs, buckets, features)
 	}
 }
+
+func TestHydrationRefreshesPortableBodyMetadata(t *testing.T) {
+	ctx := context.Background()
+	dbPath := filepath.Join(t.TempDir(), "gitcrawl.db")
+	st, err := Open(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+
+	repoID, err := st.UpsertRepository(ctx, Repository{
+		Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl",
+		RawJSON: "{}", UpdatedAt: "2026-07-29T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("repo: %v", err)
+	}
+	threadID, err := st.UpsertThread(ctx, Thread{
+		RepoID: repoID, GitHubID: "1", Number: 1, Kind: "issue", State: "open",
+		Title: "full cloud archive", Body: "old",
+		HTMLURL:    "https://github.com/openclaw/gitcrawl/issues/1",
+		LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"old"}`,
+		ContentHash: "original", UpdatedAtGitHub: "2026-07-29T00:00:00Z",
+		UpdatedAt: "2026-07-29T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("thread: %v", err)
+	}
+	if _, err := st.UpsertComment(ctx, Comment{
+		ThreadID: threadID, GitHubID: "c1", CommentType: "issue_comment",
+		Body: "old", RawJSON: "{}",
+		CreatedAtGitHub: "2026-07-29T00:00:00Z",
+	}); err != nil {
+		t.Fatalf("comment: %v", err)
+	}
+	if _, err := st.PrunePortablePayloads(ctx, PortablePruneOptions{BodyChars: 8}); err != nil {
+		t.Fatalf("prune: %v", err)
+	}
+	if err := st.Close(); err != nil {
+		t.Fatalf("close portable store: %v", err)
+	}
+
+	st, err = Open(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("reopen portable store: %v", err)
+	}
+	defer st.Close()
+
+	if _, err := st.UpsertThread(ctx, Thread{
+		RepoID: repoID, GitHubID: "1", Number: 1, Kind: "issue", State: "open",
+		Title: "full cloud archive", Body: "new body is longer",
+		HTMLURL:    "https://github.com/openclaw/gitcrawl/issues/1",
+		LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"new body is longer"}`,
+		ContentHash: "hydrated", UpdatedAtGitHub: "2026-07-29T01:00:00Z",
+		UpdatedAt: "2026-07-29T01:00:00Z",
+	}); err != nil {
+		t.Fatalf("hydrate thread: %v", err)
+	}
+	if _, err := st.UpsertComment(ctx, Comment{
+		ThreadID: threadID, GitHubID: "c1", CommentType: "issue_comment",
+		Body: "new comment is longer", RawJSON: "{}", CreatedAtGitHub: "2026-07-29T00:00:00Z",
+		UpdatedAtGitHub: "2026-07-29T01:00:00Z",
+	}); err != nil {
+		t.Fatalf("hydrate comment: %v", err)
+	}
+	insertedThreadID, err := st.UpsertThread(ctx, Thread{
+		RepoID: repoID, GitHubID: "2", Number: 2, Kind: "issue", State: "open",
+		Title: "new cloud row", Body: "inserted body is longer",
+		HTMLURL:    "https://github.com/openclaw/gitcrawl/issues/2",
+		LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"body":"inserted body is longer"}`,
+		ContentHash: "inserted", UpdatedAtGitHub: "2026-07-29T01:00:00Z",
+		UpdatedAt: "2026-07-29T01:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("insert hydrated thread: %v", err)
+	}
+	if _, err := st.UpsertComment(ctx, Comment{
+		ThreadID: insertedThreadID, GitHubID: "c2", CommentType: "issue_comment",
+		Body: "inserted comment is longer", RawJSON: "{}",
+		CreatedAtGitHub: "2026-07-29T01:00:00Z",
+	}); err != nil {
+		t.Fatalf("insert hydrated comment: %v", err)
+	}
+
+	var threadBody, threadExcerpt string
+	var threadBodyLength int
+	if err := st.DB().QueryRowContext(ctx, `
+		select body, body_excerpt, body_length from threads where id = ?
+	`, threadID).Scan(&threadBody, &threadExcerpt, &threadBodyLength); err != nil {
+		t.Fatalf("read hydrated thread: %v", err)
+	}
+	if threadBody != "new body is longer" ||
+		threadExcerpt != "new body" ||
+		threadBodyLength != len("new body is longer") {
+		t.Fatalf("hydrated thread metadata = body %q excerpt %q length %d", threadBody, threadExcerpt, threadBodyLength)
+	}
+
+	var commentBody, commentExcerpt string
+	var commentBodyLength int
+	if err := st.DB().QueryRowContext(ctx, `
+		select body, body_excerpt, body_length from comments where github_id = 'c1'
+	`).Scan(&commentBody, &commentExcerpt, &commentBodyLength); err != nil {
+		t.Fatalf("read hydrated comment: %v", err)
+	}
+	if commentBody != "new comment is longer" ||
+		commentExcerpt != "new comm" ||
+		commentBodyLength != len("new comment is longer") {
+		t.Fatalf("hydrated comment metadata = body %q excerpt %q length %d", commentBody, commentExcerpt, commentBodyLength)
+	}
+
+	var insertedThreadExcerpt, insertedCommentExcerpt string
+	if err := st.DB().QueryRowContext(ctx, `
+		select body_excerpt from threads where id = ?
+	`, insertedThreadID).Scan(&insertedThreadExcerpt); err != nil {
+		t.Fatalf("read inserted thread excerpt: %v", err)
+	}
+	if err := st.DB().QueryRowContext(ctx, `
+		select body_excerpt from comments where github_id = 'c2'
+	`).Scan(&insertedCommentExcerpt); err != nil {
+		t.Fatalf("read inserted comment excerpt: %v", err)
+	}
+	if insertedThreadExcerpt != "inserted" || insertedCommentExcerpt != "inserted" {
+		t.Fatalf(
+			"inserted excerpts = thread %q comment %q",
+			insertedThreadExcerpt,
+			insertedCommentExcerpt,
+		)
+	}
+}

internal/store/threads.go

@@ -254,6 +254,19 @@ func (s *Store) upsertThreadObservation(ctx context.Context, thread Thread, opti
 	if err != nil {
 		return UpsertThreadResult{}, fmt.Errorf("upsert thread: %w", err)
 	}
+	if s.portableThreadBodyMetadata {
+		if _, err := s.q().ExecContext(ctx, `
+			update threads
+			set body_length = length(coalesce(body, '')),
+				body_excerpt = case
+					when body is null then ''
+					else substr(body, 1, ?)
+				end
+			where id = ?
+		`, s.portableBodyChars, id); err != nil {
+			return UpsertThreadResult{}, fmt.Errorf("refresh portable thread body metadata: %w", err)
+		}
+	}
 	evidenceApplied := false
 	if !options.IncompleteEvidence {
 		evidenceApplied, err = s.reserveThreadEvidenceObservation(