fix: recover portable checkouts from stale index locks and missing source databases

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## 0.8.6 - Unreleased
 
+### Portable stores
+
+- Remove verified-stale Git index locks during ordinary reads and refreshes instead of treating the checkout as dirty forever.
+- Repair a missing portable source database through the standard reset-pull and reclone flow instead of returning a permanent stat error.
+
 ## 0.8.5 - 2026-07-26
 
 ### Portable stores

internal/cli/app_test.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -3133,6 +3134,80 @@ func TestReadCommandRefreshesPortableStore(t *testing.T) {
 	}
 }
 
+func TestRefreshPortableStoreRecoversFromStaleIndexLock(t *testing.T) {
+	if _, err := exec.LookPath("lsof"); err != nil {
+		t.Skip("lsof is required for verified stale-lock recovery")
+	}
+	ctx := context.Background()
+	dir := t.TempDir()
+	remoteDir := filepath.Join(dir, "remote")
+	checkoutDir := filepath.Join(dir, "checkout")
+	dbRel := filepath.Join("data", "openclaw__openclaw.sync.db")
+	if err := os.MkdirAll(filepath.Join(remoteDir, "data"), 0o755); err != nil {
+		t.Fatalf("mkdir remote data: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "init", "-b", "main"); err != nil {
+		t.Fatalf("git init: %v", err)
+	}
+	seedPortableThread(t, filepath.Join(remoteDir, dbRel), 1, "portable issue")
+	if err := runGit(ctx, remoteDir, "add", dbRel); err != nil {
+		t.Fatalf("git add seed: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "seed store"); err != nil {
+		t.Fatalf("git commit seed: %v", err)
+	}
+	if _, err := syncPortableStore(ctx, remoteDir, checkoutDir); err != nil {
+		t.Fatalf("clone portable store: %v", err)
+	}
+
+	lockPath := filepath.Join(checkoutDir, ".git", "index.lock")
+	if err := os.WriteFile(lockPath, nil, 0o644); err != nil {
+		t.Fatalf("write index lock: %v", err)
+	}
+	oldLock := time.Now().Add(-staleGitIndexLockAge - time.Second)
+	if err := os.Chtimes(lockPath, oldLock, oldLock); err != nil {
+		t.Fatalf("age index lock: %v", err)
+	}
+	if err := refreshPortableStoreForDB(ctx, filepath.Join(checkoutDir, dbRel)); err != nil {
+		t.Fatalf("refresh portable store: %v", err)
+	}
+	if _, err := os.Stat(lockPath); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("stale index lock should be removed, err=%v", err)
+	}
+}
+
+func TestRefreshPortableStoreStillDirtyWithoutStaleLock(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	remoteDir := filepath.Join(dir, "remote")
+	checkoutDir := filepath.Join(dir, "checkout")
+	dbRel := filepath.Join("data", "openclaw__openclaw.sync.db")
+	if err := os.MkdirAll(filepath.Join(remoteDir, "data"), 0o755); err != nil {
+		t.Fatalf("mkdir remote data: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "init", "-b", "main"); err != nil {
+		t.Fatalf("git init: %v", err)
+	}
+	seedPortableThread(t, filepath.Join(remoteDir, dbRel), 1, "portable issue")
+	if err := runGit(ctx, remoteDir, "add", dbRel); err != nil {
+		t.Fatalf("git add seed: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "seed store"); err != nil {
+		t.Fatalf("git commit seed: %v", err)
+	}
+	if _, err := syncPortableStore(ctx, remoteDir, checkoutDir); err != nil {
+		t.Fatalf("clone portable store: %v", err)
+	}
+
+	checkoutDB := filepath.Join(checkoutDir, dbRel)
+	if err := os.WriteFile(checkoutDB, []byte("dirty portable db"), 0o644); err != nil {
+		t.Fatalf("modify checkout db: %v", err)
+	}
+	if err := refreshPortableStoreForDB(ctx, checkoutDB); !errors.Is(err, errPortableStoreDirty) {
+		t.Fatalf("refresh dirty portable store err=%v, want %v", err, errPortableStoreDirty)
+	}
+}
+
 func TestReadCommandUsesCachedPortableStoreWhenRefreshFails(t *testing.T) {
 	ctx := context.Background()
 	dir := t.TempDir()
@@ -3377,6 +3452,113 @@ func TestPortableRuntimePropagatesNonCorruptionSourceErrors(t *testing.T) {
 	}
 }
 
+func TestPortableRuntimeRepairsMissingSourceDB(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	remoteDir := filepath.Join(dir, "remote")
+	checkoutDir := filepath.Join(dir, "checkout")
+	dbRel := filepath.Join("data", "openclaw__openclaw.sync.db")
+	if err := os.MkdirAll(filepath.Join(remoteDir, "data"), 0o755); err != nil {
+		t.Fatalf("mkdir remote data: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "init", "-b", "main"); err != nil {
+		t.Fatalf("git init: %v", err)
+	}
+	seedPortableThread(t, filepath.Join(remoteDir, dbRel), 1, "portable issue")
+	if err := runGit(ctx, remoteDir, "add", dbRel); err != nil {
+		t.Fatalf("git add seed: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "seed store"); err != nil {
+		t.Fatalf("git commit seed: %v", err)
+	}
+	if _, err := syncPortableStore(ctx, remoteDir, checkoutDir); err != nil {
+		t.Fatalf("clone portable store: %v", err)
+	}
+
+	checkoutDB := filepath.Join(checkoutDir, dbRel)
+	mirrorPath := filepath.Join(dir, "runtime", dbRel)
+	if err := os.Remove(checkoutDB); err != nil {
+		t.Fatalf("remove checkout db: %v", err)
+	}
+	changed, err := refreshPortableRuntimeDB(ctx, checkoutDB, mirrorPath, false, filepath.Join(dir, "config.toml"))
+	if err != nil {
+		t.Fatalf("refresh portable runtime db: %v", err)
+	}
+	if !changed {
+		t.Fatal("missing source repair should refresh the runtime mirror")
+	}
+	if err := sqliteStoreHealth(ctx, checkoutDB); err != nil {
+		t.Fatalf("checkout db should be restored and healthy: %v", err)
+	}
+	if err := sqliteStoreHealth(ctx, mirrorPath); err != nil {
+		t.Fatalf("runtime mirror should be created and healthy: %v", err)
+	}
+}
+
+func TestPortableRuntimeServesHealthyMirrorWhenSourceUnrecoverable(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	remoteDir := filepath.Join(dir, "remote")
+	checkoutDir := filepath.Join(dir, "checkout")
+	dbRel := filepath.Join("data", "openclaw__openclaw.sync.db")
+	// The remote never contained the database, so repair and reclone both
+	// complete without restoring it; a healthy mirror must keep serving.
+	if err := os.MkdirAll(filepath.Join(remoteDir, "data"), 0o755); err != nil {
+		t.Fatalf("mkdir remote data: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(remoteDir, "data", ".gitkeep"), nil, 0o644); err != nil {
+		t.Fatalf("write gitkeep: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "init", "-b", "main"); err != nil {
+		t.Fatalf("git init: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "add", "."); err != nil {
+		t.Fatalf("git add: %v", err)
+	}
+	if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "empty store"); err != nil {
+		t.Fatalf("git commit: %v", err)
+	}
+	if _, err := syncPortableStore(ctx, remoteDir, checkoutDir); err != nil {
+		t.Fatalf("clone portable store: %v", err)
+	}
+
+	checkoutDB := filepath.Join(checkoutDir, dbRel)
+	mirrorPath := filepath.Join(dir, "runtime", dbRel)
+	seedPortableThread(t, mirrorPath, 1, "healthy mirror content")
+
+	changed, err := refreshPortableRuntimeDB(ctx, checkoutDB, mirrorPath, false, filepath.Join(dir, "config.toml"))
+	if err != nil {
+		t.Fatalf("healthy mirror should keep serving when the source is unrecoverable: %v", err)
+	}
+	if changed {
+		t.Fatal("unrecoverable source must not report a refreshed mirror")
+	}
+	if err := sqliteStoreHealth(ctx, mirrorPath); err != nil {
+		t.Fatalf("mirror should remain healthy: %v", err)
+	}
+
+	countBackups := func() int {
+		entries, err := os.ReadDir(filepath.Join(dir, "backups"))
+		if err != nil {
+			if os.IsNotExist(err) {
+				return 0
+			}
+			t.Fatalf("read backups dir: %v", err)
+		}
+		return len(entries)
+	}
+	backupsAfterFirst := countBackups()
+	if backupsAfterFirst == 0 {
+		t.Fatal("first recovery attempt should have run repair/reclone")
+	}
+	if changed, err := refreshPortableRuntimeDB(ctx, checkoutDB, mirrorPath, false, filepath.Join(dir, "config.toml")); err != nil || changed {
+		t.Fatalf("backed-off retry should serve the mirror quietly, changed=%v err=%v", changed, err)
+	}
+	if got := countBackups(); got != backupsAfterFirst {
+		t.Fatalf("recovery must back off instead of repeating per read: backups %d -> %d", backupsAfterFirst, got)
+	}
+}
+
 func TestPortableRuntimeRejectsManifestMismatchBeforeReplacingMirror(t *testing.T) {
 	ctx := context.Background()
 	dir := t.TempDir()

internal/cli/runtime.go

@@ -47,6 +47,7 @@ const portableStoreRepairTimeout = 90 * time.Second
 const portableStoreRefreshTTL = 2 * time.Minute
 const portableStoreRefreshFailureBackoff = time.Minute
 const portableRuntimeTempMaxAge = time.Hour
+const portableSourceRecoveryBackoff = 15 * time.Minute
 const portableStoreMarkerFile = "gitcrawl-portable-store"
 const staleGitIndexLockAge = 2 * time.Second
 
@@ -136,12 +137,19 @@ func refreshPortableStoreForDB(ctx context.Context, dbPath string) error {
 	if !ok {
 		return nil
 	}
-	if !gitWorktreeClean(ctx, root) {
+	clean := gitWorktreeClean(ctx, root)
+	if !clean {
+		removed, _ := removeStaleGitIndexLock(ctx, root, staleGitIndexLockAge)
+		if removed {
+			clean = gitWorktreeClean(ctx, root)
+		}
+	}
+	if !clean {
 		return errPortableStoreDirty
 	}
 	pullCtx, cancel := context.WithTimeout(ctx, portableStoreRefreshTimeout)
 	defer cancel()
-	if err := fastForwardGitCheckout(pullCtx, root, true); err != nil {
+	if _, err := fastForwardGitCheckoutWithStaleIndexLockRetry(pullCtx, root, true); err != nil {
 		return err
 	}
 	return removePortableSQLiteSidecars(root)
@@ -275,10 +283,33 @@ func refreshPortableRuntimeDB(ctx context.Context, sourceDBPath, mirrorPath stri
 		_ = refreshPortableStoreForDBIfDue(ctx, sourceDBPath, mirrorPath)
 	}
 	needsCopy, err := portableRuntimeNeedsCopy(sourceDBPath, mirrorPath)
+	statePath := portableStoreRefreshStatePath(mirrorPath)
 	if err != nil {
-		return false, err
+		if !isRepairablePortableSource || !errors.Is(err, os.ErrNotExist) {
+			return false, err
+		}
+		// A recovered source is preferred, but any failure along the
+		// repair/reclone chain degrades to serving a healthy mirror so a
+		// missing source cannot take reads down. Recovery runs reset/pull and
+		// potentially a full reclone, so attempts are backed off via the
+		// recorded repair timestamp instead of repeating on every read. The
+		// backoff only gates this stat-failure branch: an externally restored
+		// source makes the stat succeed and skips the gate entirely.
+		recoverErr := err
+		state := readPortableStoreRefreshState(statePath)
+		if !recentPortableRefresh(state.LastRepairAt, time.Now().UTC(), portableSourceRecoveryBackoff) {
+			recoverErr = recoverMissingPortableSource(ctx, sourceDBPath, configPath, statePath)
+			if recoverErr == nil {
+				needsCopy, recoverErr = portableRuntimeNeedsCopy(sourceDBPath, mirrorPath)
+			}
+		}
+		if recoverErr != nil {
+			if mirrorHealthErr := sqliteStoreHealth(ctx, mirrorPath); mirrorHealthErr == nil {
+				return false, nil
+			}
+			return false, recoverErr
+		}
 	}
-	statePath := portableStoreRefreshStatePath(mirrorPath)
 	mirrorCorrupt := false
 	if isRepairablePortableSource && !needsCopy {
 		mirrorHealthErr := portableMirrorCachedHealth(ctx, mirrorPath, sourceDBPath, statePath)
@@ -348,6 +379,22 @@ type portableStoreRefreshState struct {
 	LastRepairError             string `json:"last_repair_error,omitempty"`
 }
 
+func recoverMissingPortableSource(ctx context.Context, sourceDBPath, configPath, statePath string) error {
+	repair, err := repairMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)
+	recordPortableRepairState(statePath, repair, err)
+	if err != nil {
+		return fmt.Errorf("repair malformed portable store db: %w", err)
+	}
+	if _, statErr := os.Stat(sourceDBPath); errors.Is(statErr, os.ErrNotExist) {
+		reclone, recloneErr := recloneMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)
+		recordPortableRepairState(statePath, reclone, recloneErr)
+		if recloneErr != nil {
+			return fmt.Errorf("reclone malformed portable store db: %w", recloneErr)
+		}
+	}
+	return nil
+}
+
 func refreshPortableStoreForDBIfDue(ctx context.Context, sourceDBPath, mirrorPath string) error {
 	ttl := portableStoreRefreshInterval()
 	statePath := portableStoreRefreshStatePath(mirrorPath)