status: fix FSMonitor history and clean-proof gaps

Documentation/technical/status-clean-proof.adoc

@@ -83,9 +83,12 @@ checksum is accepted only when the pinned index is bound by the durable
 local-APFS identity used for raced-input checks.
 
 The sidecar is installed while the index lock remains held and after
-the pinned index is rechecked. Status then rolls back the index lock, so
-issuing a sidecar does not itself rewrite the index. With optional locks
-disabled, status does not issue a sidecar.
+the pinned index is rechecked. If the exact query first has to repair
+the index's file system monitor checkpoint, status writes that repair,
+refreshes the resumable history checkpoint, and then installs a proof
+bound to the rewritten index. Status rolls back the lock used for the
+sidecar itself, so issuing a sidecar does not itself rewrite the index.
+With optional locks disabled, status does not issue a sidecar.
 
 Validation and races
 --------------------

Makefile

@@ -2580,6 +2580,8 @@ LIBS = $(filter-out %.o, $(GITLIBS)) $(EXTLIBS)
 
 BASIC_CFLAGS += $(COMPAT_CFLAGS)
 LIB_OBJS += $(COMPAT_OBJS)
+# Keep provisional-index glue after every existing library object.
+LIB_OBJS += clean-status-index-provisional.o
 
 # Quote for C
 

builtin/add.c

@@ -29,6 +29,7 @@
 #include "submodule.h"
 #include "add-interactive.h"
 #include "merge-ll.h"
+#include "wt-status.h"
 
 static const char * const builtin_add_usage[] = {
 	N_("git add [<options>] [--] <pathspec>..."),
@@ -467,6 +468,7 @@ int cmd_add(int argc,
 	struct dir_struct dir = DIR_INIT;
 	int flags;
 	int add_new_files;
+	int had_full_proof = 0;
 	int preserve_add_history = 0;
 	int require_pathspec;
 	char *seen = NULL;
@@ -591,7 +593,7 @@ int cmd_add(int argc,
 	if (refresh_only) {
 		clean_status_enable_external_history(repo);
 		clean_status_set_config_digest(repo, &clean_digest);
-	} else if (!show_only && !intent_to_add && !add_renormalize &&
+	} else if (!show_only && !add_renormalize &&
 		   !chmod_arg && !include_sparse && !ignore_add_errors) {
 		preserve_add_history = 1;
 		flags |= ADD_CACHE_TRACK_CLEAN_HISTORY;
@@ -601,6 +603,8 @@ int cmd_add(int argc,
 
 	if (repo_read_index_preload(repo, &pathspec, 0) < 0)
 		die(_("index file corrupt"));
+	had_full_proof =
+		clean_status_has_persistent_fsmonitor_semantic_history(repo->index);
 	if (preserve_add_history &&
 	    (repo->index->split_index || repo->index->sparse_index))
 		clean_status_invalidate_current_proof(repo->index);
@@ -717,6 +721,12 @@ int cmd_add(int argc,
 finish:
 	if (preserve_add_history && exit_status)
 		clean_status_invalidate_current_proof(repo->index);
+	else if (preserve_add_history && !show_only && !intent_to_add &&
+		 (!clean_status_has_current_full_fsmonitor_proof(repo->index) ||
+		  !repo->index->fsmonitor_untracked_valid) &&
+		 wt_status_repair_fsmonitor_proof_after_index_update(
+			 repo, &lock_file, had_full_proof) < 0)
+		die(_("unable to repair new index file"));
 	if (show_only)
 		rollback_lock_file(&lock_file);
 	else if (write_locked_index(repo->index, &lock_file,

builtin/checkout.c

@@ -554,6 +554,7 @@ static int checkout_paths(const struct checkout_opts *opts,
 	int checkout_index;
 	int preserve_source_tree_history = 0;
 	int source_tree_index_changed = 0;
+	int repair_after_checkout = 0;
 
 	trace2_cmd_mode(opts->patch_mode ? "patch" : "path");
 
@@ -678,6 +679,9 @@ static int checkout_paths(const struct checkout_opts *opts,
 	}
 	if (repo_read_index_preload(the_repository, &opts->pathspec, 0) < 0)
 		return error(_("index file corrupt"));
+	repair_after_checkout = opts->checkout_worktree &&
+		clean_status_has_current_full_fsmonitor_proof(
+			the_repository->index);
 
 	if (preserve_source_tree_history &&
 	    (the_repository->index->split_index ||
@@ -764,6 +768,10 @@ static int checkout_paths(const struct checkout_opts *opts,
 		if (!the_repository->index->cache_changed &&
 		    !hook_exists(the_repository, "post-index-change"))
 			flags |= SKIP_IF_UNCHANGED;
+		if (wt_status_repair_fsmonitor_proof_after_worktree_update(
+			    the_repository, &lock_file,
+			    repair_after_checkout) < 0)
+			die(_("unable to repair new index file"));
 		if (write_locked_index(the_repository->index, &lock_file, flags))
 			die(_("unable to write new index file"));
 	} else {
@@ -836,6 +844,9 @@ static int reset_tree(struct tree *tree, const struct checkout_opts *o,
 	opts.verbose_update = o->show_progress;
 	opts.src_index = the_repository->index;
 	opts.dst_index = the_repository->index;
+	opts.preserve_semantic_history = worktree &&
+		clean_status_revalidated_token_matches(the_repository->index);
+	opts.preserve_untracked_history = opts.preserve_semantic_history;
 	init_checkout_metadata(&opts.meta, info->refname,
 			       info->commit ? &info->commit->object.oid : null_oid(the_hash_algo),
 			       NULL);
@@ -907,7 +918,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 			      bool quiet,
 			      int *writeout_error)
 {
-	int ret;
+	int ret, repair_after_checkout;
 	struct lock_file lock_file = LOCK_INIT;
 	struct tree *new_tree;
 
@@ -927,6 +938,9 @@ static int merge_working_tree(const struct checkout_opts *opts,
 		rollback_lock_file(&lock_file);
 		return error(_("index file corrupt"));
 	}
+	repair_after_checkout =
+		clean_status_has_current_full_fsmonitor_proof(
+			the_repository->index);
 
 	resolve_undo_clear_index(the_repository->index);
 	if (opts->new_orphan_branch && opts->orphan_from_empty_tree) {
@@ -969,6 +983,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 		init_topts(&topts, opts->show_progress,
 			   opts->overwrite_ignore, quiet);
 		topts.preserve_semantic_history = 1;
+		topts.preserve_untracked_history = 1;
 		init_checkout_metadata(&topts.meta, new_branch_info->refname,
 				       new_branch_info->commit ?
 				       &new_branch_info->commit->object.oid :
@@ -1001,6 +1016,9 @@ static int merge_working_tree(const struct checkout_opts *opts,
 
 	if (!cache_tree_fully_valid(the_repository->index->cache_tree))
 		cache_tree_update(the_repository->index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR);
+	if (wt_status_repair_fsmonitor_proof_after_worktree_update(
+		    the_repository, &lock_file, repair_after_checkout) < 0)
+		die(_("unable to repair new index file"));
 
 	if (write_locked_index(the_repository->index, &lock_file, COMMIT_LOCK))
 		die(_("unable to write new index file"));

builtin/commit.c

@@ -1898,21 +1898,35 @@ struct repository *repo UNUSED)
 	int default_status_command = argc == 1 && (!prefix || !*prefix);
 	int exact_clean_command = argc == 2 &&
 		!strcmp(argv[1], "--porcelain=v2") && (!prefix || !*prefix);
+	int short_clean_command = argc == 2 &&
+		(!strcmp(argv[1], "--short") || !strcmp(argv[1], "-s")) &&
+		(!prefix || !*prefix);
 	int exact_clean_query;
 	int normal_clean_query;
+	int short_clean_query;
+	int certifying_clean_query;
 	int reusable_clean_query;
 	int normal_has_head;
 	int reissue_clean_sidecar = 0;
 	int repository_inputs_changed = 0;
 	int sidecar_provider_reset = 0;
 	int reissue_after_write = 0;
+	int issue_exact_after_write = 0;
+	int issue_certifying_after_write = 0;
+	int reload_racy_after_write = 0;
+	int exact_after_write_candidate = 0;
 	int save_history_after_write = 0;
+	int postwrite_sidecar_candidate = 0;
+	int postwrite_sidecar_pinned = 0;
 	int deferred_scoped_history = 0;
 	int guarded_scoped_history_source = 0;
 	int optional_status_writes;
 	struct clean_status_index_snapshot scoped_history_source = {
 		.fd = -1,
 	};
+	struct clean_status_index_snapshot postwrite_sidecar_source = {
+		.fd = -1,
+	};
 	struct clean_status_index_write_receipt written_index =
 		CLEAN_STATUS_INDEX_WRITE_RECEIPT_INIT;
 	struct object_id oid;
@@ -2018,6 +2032,14 @@ struct repository *repo UNUSED)
 		!s.submodule_summary &&
 		s.show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES &&
 		!repo_config_values(the_repository)->apply_sparse_checkout;
+	short_clean_query = short_clean_command &&
+		status_format == STATUS_FORMAT_SHORT && normal_has_head &&
+		!s.pathspec.nr && !s.show_branch && !s.show_stash &&
+		!s.show_ignored_mode && !s.null_termination && !s.verbose &&
+		!s.submodule_summary &&
+		s.show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES &&
+		!repo_config_values(the_repository)->apply_sparse_checkout;
+	certifying_clean_query = normal_clean_query || short_clean_query;
 	reusable_clean_query = normal_has_head &&
 		!s.show_ignored_mode && !s.submodule_summary &&
 		/* A clean merge still prints a staged-changes header with -vv. */
@@ -2028,7 +2050,7 @@ struct repository *repo UNUSED)
 		!s.submodule_summary &&
 		!repo_config_values(the_repository)->apply_sparse_checkout;
 	clean_status_enable_external_history(the_repository);
-	s.certify_clean_status = exact_clean_query;
+	s.certify_clean_status = exact_clean_query || short_clean_query;
 	if (reusable_clean_query &&
 	    clean_status_try_sidecar(the_repository, &clean_digest,
 				     &repository_inputs_changed,
@@ -2039,7 +2061,7 @@ struct repository *repo UNUSED)
 			return 0;
 		}
 	}
-	if (normal_clean_query && optional_status_writes &&
+	if (certifying_clean_query && optional_status_writes &&
 	    clean_status_identity_is_durable())
 		reissue_clean_sidecar =
 			clean_status_sidecar_needs_reissue(
@@ -2089,7 +2111,7 @@ struct repository *repo UNUSED)
 			clean_status_capture_external_history_source(
 				the_repository->index);
 	}
-	if (normal_clean_query && optional_status_writes &&
+	if (certifying_clean_query && optional_status_writes &&
 	    clean_status_identity_is_durable() &&
 	    (reissue_clean_sidecar ||
 	     clean_status_external_history_was_restored(
@@ -2163,6 +2185,8 @@ struct repository *repo UNUSED)
 				the_repository->index);
 		int external_saved = 0;
 		int persist_restored_boundary = 0;
+		int racy_fresh_history = !external_restored &&
+			has_racy_timestamp(the_repository->index);
 		int preserve_entry_changes =
 			(!external_restored &&
 			 (the_repository->index->cache_changed & CE_ENTRY_CHANGED)) ||
@@ -2192,6 +2216,11 @@ struct repository *repo UNUSED)
 		else if (deferred_history &&
 			 !hook_exists(the_repository, "post-index-change"))
 			save_history_after_write = 1;
+		if (certifying_clean_query && racy_fresh_history &&
+		    !s.change.nr && !s.untracked.nr && !s.ignored.nr &&
+		    !external_saved &&
+		    !hook_exists(the_repository, "post-index-change"))
+			save_history_after_write = reload_racy_after_write = 1;
 		if (external_restored && !external_saved &&
 		    clean_status_external_history_owns_index(
 			    the_repository->index) &&
@@ -2200,10 +2229,21 @@ struct repository *repo UNUSED)
 			trace2_data_intmax("fsmonitor", the_repository,
 					   "history/external-racy-index-persisted", 1);
 		}
-		reissue_after_write = normal_clean_query &&
+		reissue_after_write = certifying_clean_query &&
 			reissue_clean_sidecar && preserve_entry_changes &&
 			!external_restored && !persist_restored_boundary &&
 			!hook_exists(the_repository, "post-index-change");
+		/*
+		 * An exact query may have completed a clean scan while repairing
+		 * the provider checkpoint or cached stat data.  Bind its proof to
+		 * the repaired index, after the resumable history is durable.
+		 */
+		exact_after_write_candidate = exact_clean_query &&
+			preserve_entry_changes && !external_restored &&
+			!persist_restored_boundary &&
+			!hook_exists(the_repository, "post-index-change");
+		issue_exact_after_write =
+			exact_after_write_candidate && external_saved;
 
 		if (the_repository->index->fsmonitor_legacy_untracked_fallback &&
 		    !preserve_entry_changes && !external_saved) {
@@ -2212,7 +2252,7 @@ struct repository *repo UNUSED)
 		} else if (exact_clean_query) {
 			if (!preserve_entry_changes && external_saved &&
 			    clean_status_issue_sidecar(
-				    &s, &clean_digest, &index_lock, 0))
+				    &s, &clean_digest, &index_lock, NULL, 0))
 				fd = -1;
 			else if (!preserve_entry_changes &&
 				 !persist_restored_boundary &&
@@ -2222,10 +2262,10 @@ struct repository *repo UNUSED)
 			}
 		} else if (!preserve_entry_changes &&
 			   !persist_restored_boundary &&
-			   normal_clean_query &&
+			   certifying_clean_query &&
 			   (external_restored || reissue_clean_sidecar) &&
 			   clean_status_issue_sidecar(
-				   &s, &clean_digest, &index_lock, 1)) {
+				   &s, &clean_digest, &index_lock, NULL, 1)) {
 			fd = -1;
 		} else if (!preserve_entry_changes &&
 			   !persist_restored_boundary &&
@@ -2243,30 +2283,68 @@ struct repository *repo UNUSED)
 				   "history/scoped-source-epoch-mismatch", 1);
 	}
 	if (0 <= fd) {
+		postwrite_sidecar_candidate =
+			reissue_after_write || issue_exact_after_write ||
+			save_history_after_write;
 		repo_update_index_if_able_with_receipt(the_repository, &index_lock,
 						      &written_index);
+		if (postwrite_sidecar_candidate)
+			postwrite_sidecar_pinned =
+				!clean_status_sidecar_postwrite_test_barrier();
 		clean_status_index_adopt_write_receipt(the_repository->index,
 						      &written_index);
+		if (postwrite_sidecar_pinned &&
+		    clean_status_index_snapshot_pin(
+			    &postwrite_sidecar_source,
+			    the_repository->index)) {
+			postwrite_sidecar_pinned = 0;
+			trace2_data_string("status", the_repository,
+					   "clean-proof/miss",
+					   "postwrite-index-raced");
+		}
+		if (reload_racy_after_write) {
+			/*
+			 * Re-read the committed index before checkpointing a fresh racy
+			 * checkout. The old in-memory epoch cannot authenticate the new
+			 * file even though this status established its clean contents.
+			 */
+			discard_index(the_repository->index);
+			if (repo_read_index(the_repository) < 0)
+				save_history_after_write = 0;
+		}
 		if (save_history_after_write &&
 		    !hook_exists(the_repository, "post-index-change") &&
 		    repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) {
 			if (clean_status_save_external_history(
-				    the_repository->index))
+				    the_repository->index)) {
 				trace2_data_intmax("fsmonitor", the_repository,
 						   "history/external-postwrite-stored", 1);
+				if (exact_after_write_candidate)
+					issue_exact_after_write = 1;
+				else if (certifying_clean_query)
+					issue_certifying_after_write = 1;
+			}
 			rollback_lock_file(&index_lock);
 		}
-		if (reissue_after_write &&
+		if (postwrite_sidecar_pinned &&
+		    (reissue_after_write || issue_exact_after_write ||
+		     issue_certifying_after_write) &&
 		    repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) {
 			if (clean_status_issue_sidecar(
-				    &s, &clean_digest, &index_lock, 1))
+				    &s, &clean_digest, &index_lock,
+				    &postwrite_sidecar_source,
+				    reissue_after_write ||
+				    issue_certifying_after_write))
 				trace2_data_intmax("status", the_repository,
-						   "clean-proof/postwrite-reissued", 1);
+						   reissue_after_write ?
+						   "clean-proof/postwrite-reissued" :
+						   "clean-proof/postwrite-issued", 1);
 			else
 				rollback_lock_file(&index_lock);
 		}
 	}
 	clean_status_index_write_receipt_release(&written_index);
+	clean_status_index_snapshot_release(&postwrite_sidecar_source);
 	clean_status_index_snapshot_release(&scoped_history_source);
 
 	if (s.relative_paths)