diff --git a/resources/flashgrep/README.md b/resources/flashgrep/README.md index c4d8429161..7d84998cda 100644 --- a/resources/flashgrep/README.md +++ b/resources/flashgrep/README.md @@ -2,7 +2,7 @@ Place the prebuilt `flashgrep` daemon binary in this directory. Pinned release: -- `v0.2.10` from `wgqqqqq/flashgrep` +- `v0.2.15` from `wgqqqqq/flashgrep` Expected filenames: diff --git a/resources/flashgrep/VERSION.json b/resources/flashgrep/VERSION.json index 42da02c4b5..5436c66097 100644 --- a/resources/flashgrep/VERSION.json +++ b/resources/flashgrep/VERSION.json @@ -1,5 +1,5 @@ { "repo": "wgqqqqq/flashgrep", - "tag": "v0.2.10", - "published_at": "2026-06-17T08:15:11Z" + "tag": "v0.2.15", + "published_at": "2026-08-18T07:47:30Z" } diff --git a/resources/flashgrep/flashgrep-aarch64-apple-darwin b/resources/flashgrep/flashgrep-aarch64-apple-darwin index 410b228d2d..b80a840378 100755 Binary files a/resources/flashgrep/flashgrep-aarch64-apple-darwin and b/resources/flashgrep/flashgrep-aarch64-apple-darwin differ diff --git a/resources/flashgrep/flashgrep-aarch64-pc-windows-msvc.exe b/resources/flashgrep/flashgrep-aarch64-pc-windows-msvc.exe index d99a4b41bd..266fb63040 100644 Binary files a/resources/flashgrep/flashgrep-aarch64-pc-windows-msvc.exe and b/resources/flashgrep/flashgrep-aarch64-pc-windows-msvc.exe differ diff --git a/resources/flashgrep/flashgrep-aarch64-unknown-linux-musl b/resources/flashgrep/flashgrep-aarch64-unknown-linux-musl index 3639f94f22..1e9923e58f 100755 Binary files a/resources/flashgrep/flashgrep-aarch64-unknown-linux-musl and b/resources/flashgrep/flashgrep-aarch64-unknown-linux-musl differ diff --git a/resources/flashgrep/flashgrep-x86_64-apple-darwin b/resources/flashgrep/flashgrep-x86_64-apple-darwin index 53231225ad..607973a9fd 100755 Binary files a/resources/flashgrep/flashgrep-x86_64-apple-darwin and b/resources/flashgrep/flashgrep-x86_64-apple-darwin differ diff --git a/resources/flashgrep/flashgrep-x86_64-pc-windows-msvc.exe b/resources/flashgrep/flashgrep-x86_64-pc-windows-msvc.exe index b74384f85c..ba09667c58 100644 Binary files a/resources/flashgrep/flashgrep-x86_64-pc-windows-msvc.exe and b/resources/flashgrep/flashgrep-x86_64-pc-windows-msvc.exe differ diff --git a/resources/flashgrep/flashgrep-x86_64-unknown-linux-musl b/resources/flashgrep/flashgrep-x86_64-unknown-linux-musl index 51aa97b8a0..75fb69b3e8 100755 Binary files a/resources/flashgrep/flashgrep-x86_64-unknown-linux-musl and b/resources/flashgrep/flashgrep-x86_64-unknown-linux-musl differ diff --git a/scripts/core-boundaries/rules/source/required-rules.mjs b/scripts/core-boundaries/rules/source/required-rules.mjs index 80e882a646..aa72abf9ba 100644 --- a/scripts/core-boundaries/rules/source/required-rules.mjs +++ b/scripts/core-boundaries/rules/source/required-rules.mjs @@ -8424,24 +8424,20 @@ export const requiredContentRules = [ regex: /\bpub struct WorkspaceSearchRepoConfig\b/, message: 'missing stable workspace-search repo config contract', }, - { - regex: /\bwith_scan_fallback\b/, - message: 'missing flashgrep scan fallback request flag', - }, ], }, { path: 'src/crates/services/services-integrations/src/workspace_search/result_mapping.rs', reason: - 'services-integrations workspace_search result mapping must own shared flashgrep preview/result conversion', + 'services-integrations workspace_search result mapping must own flashgrep result conversion and delegate content previews to line_hydration', patterns: [ { regex: /\bconvert_hits_to_file_search_results\b/, message: 'missing hit-to-file-result conversion owner', }, { - regex: /\bsplit_preview\b/, - message: 'missing preview split contract', + regex: /\bline_hydration\b/, + message: 'missing preview hydration ownership note', }, { regex: /\bpreview_inside\b/, @@ -8449,6 +8445,44 @@ export const requiredContentRules = [ }, ], }, + { + path: 'src/crates/services/services-integrations/src/workspace_search/line_hydration.rs', + reason: + 'services-integrations workspace_search must own disk hydration of daemon line positions, because flashgrep never returns line text', + patterns: [ + { + regex: /\bpub\(crate\)\s+fn\s+hydrate_grouped_line_matches\b/, + message: 'missing grouped line-match hydration owner', + }, + { + regex: /\bMAX_HYDRATED_LINE_COLUMNS\b/, + message: 'missing long-line clamp shared with the ripgrep path', + }, + { + regex: /\bContentMatchPreviewBuilder\b/, + message: 'previews must be built with the shared services-core builder', + }, + ], + }, + { + path: 'src/crates/services/services-core/src/filesystem/content_preview.rs', + reason: + 'services-core filesystem must own the content preview primitives shared by the ripgrep and indexed search paths', + patterns: [ + { + regex: /\bpub fn compile_content_search_regex\b/, + message: 'missing shared content-search regex compiler', + }, + { + regex: /\bpub fn build_content_match_preview\b/, + message: 'missing shared content preview builder', + }, + { + regex: /\bpub struct ContentMatchPreviewBuilder\b/, + message: 'missing reusable preview builder for batched hydration', + }, + ], + }, { path: 'src/crates/assembly/core/src/service/search/service.rs', reason: @@ -8532,10 +8566,6 @@ export const requiredContentRules = [ regex: /\bensure_remote_search_context\b/, message: 'missing remote search context lifecycle owner', }, - { - regex: /\ballow_scan_fallback:\s*true\b/, - message: 'missing remote scan fallback contract', - }, { regex: /\bfallback_query\b/, message: 'missing FilesWithMatches fallback query', diff --git a/scripts/core-boundaries/self-test.mjs b/scripts/core-boundaries/self-test.mjs index 9d03e78a56..e5e6733c82 100644 --- a/scripts/core-boundaries/self-test.mjs +++ b/scripts/core-boundaries/self-test.mjs @@ -4395,11 +4395,19 @@ export function runManifestParserSelfTest({ }, { path: 'src/crates/services/services-integrations/src/workspace_search/service.rs', - contracts: ['WorkspaceSearchRepoConfig', 'with_scan_fallback'], + contracts: ['WorkspaceSearchRepoConfig'], }, { path: 'src/crates/services/services-integrations/src/workspace_search/result_mapping.rs', - contracts: ['convert_hits_to_file_search_results', 'split_preview', 'preview_inside'], + contracts: ['convert_hits_to_file_search_results', 'line_hydration', 'preview_inside'], + }, + { + path: 'src/crates/services/services-integrations/src/workspace_search/line_hydration.rs', + contracts: ['hydrate_grouped_line_matches', 'MAX_HYDRATED_LINE_COLUMNS', 'ContentMatchPreviewBuilder'], + }, + { + path: 'src/crates/services/services-core/src/filesystem/content_preview.rs', + contracts: ['compile_content_search_regex', 'build_content_match_preview', 'ContentMatchPreviewBuilder'], }, { path: 'src/crates/assembly/core/src/service/search/service.rs', @@ -4423,7 +4431,7 @@ export function runManifestParserSelfTest({ }, { path: 'src/crates/services/services-integrations/src/remote_ssh/workspace_search/service.rs', - contracts: ['RemoteWorkspaceSearchProvider', 'RemoteWorkspaceSearchService', 'RemoteWorkspaceSearchStdioProtocol', 'REMOTE_STDIO_SESSIONS', 'ensure_remote_search_context', 'allow_scan_fallback', 'fallback_query', 'remote_search_rejects_non_linux_before_stdio_open'], + contracts: ['RemoteWorkspaceSearchProvider', 'RemoteWorkspaceSearchService', 'RemoteWorkspaceSearchStdioProtocol', 'REMOTE_STDIO_SESSIONS', 'ensure_remote_search_context', 'fallback_query', 'remote_search_rejects_non_linux_before_stdio_open'], }, { path: 'src/crates/assembly/core/src/service/search/mod.rs', diff --git a/src/apps/desktop/src/api/app_state.rs b/src/apps/desktop/src/api/app_state.rs index 1be7861fcd..562d15f2d0 100644 --- a/src/apps/desktop/src/api/app_state.rs +++ b/src/apps/desktop/src/api/app_state.rs @@ -1,6 +1,8 @@ //! Application state management -use crate::api::workspace_activation::spawn_workspace_background_warmup; +use crate::api::workspace_activation::{ + spawn_restored_workspace_auto_index, spawn_workspace_background_warmup, +}; use bitfun_core::agentic::side_question::SideQuestionRuntime; use bitfun_core::agentic::{agents, tools}; use bitfun_core::infrastructure::ai::{AIClient, AIClientFactory}; @@ -15,7 +17,7 @@ use bitfun_core::util::errors::*; use bitfun_services_integrations::speech::{SpeechService, SpeechStoragePaths}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use thiserror::Error; @@ -208,9 +210,33 @@ impl AppState { })); let initial_workspace = workspace_service.get_current_workspace().await; + let mut restored_workspaces = workspace_service.get_opened_workspaces().await; + if let Some(initial_workspace) = initial_workspace.as_ref() { + if !restored_workspaces + .iter() + .any(|workspace| workspace.id == initial_workspace.id) + { + restored_workspaces.push(initial_workspace.clone()); + } + } let initial_workspace_path = initial_workspace .as_ref() .map(|workspace| workspace.root_path.clone()); + let mut index_budget_roots = workspace_service + .get_recent_workspaces() + .await + .into_iter() + .filter(|workspace| workspace.workspace_kind != workspace::WorkspaceKind::Remote) + .map(|workspace| workspace.root_path) + .collect::>(); + let mut known_index_roots = index_budget_roots.iter().cloned().collect::>(); + for workspace in workspace_service.list_workspace_infos().await { + if workspace.workspace_kind != workspace::WorkspaceKind::Remote + && known_index_roots.insert(workspace.root_path.clone()) + { + index_budget_roots.push(workspace.root_path); + } + } // Initialize SSH Remote services synchronously so they're ready before app starts let ssh_data_dir = dirs::data_local_dir() @@ -320,6 +346,7 @@ impl AppState { if let Some(workspace_info) = initial_workspace { spawn_workspace_background_warmup(&app_state, workspace_info); } + spawn_restored_workspace_auto_index(&app_state, restored_workspaces, index_budget_roots); log::info!("AppState initialized successfully"); Ok(app_state) diff --git a/src/apps/desktop/src/api/search_api.rs b/src/apps/desktop/src/api/search_api.rs index 1439e3924a..07a4e91168 100644 --- a/src/apps/desktop/src/api/search_api.rs +++ b/src/apps/desktop/src/api/search_api.rs @@ -21,7 +21,11 @@ pub struct SearchRepoIndexRequest { pub struct SearchMetadataResponse { pub backend: WorkspaceSearchBackend, pub repo_phase: WorkspaceSearchRepoPhase, - pub rebuild_recommended: bool, + pub base_advance_in_progress: bool, + /// `true` when the daemon still owed a worktree reconcile at query time, so these results + /// describe the worktree as of the last observation. No query path waits for that reconcile — + /// stale-but-instant beats correct-but-blocked — so this is reported rather than waited out. + pub workspace_probe_pending: bool, pub candidate_docs: usize, pub matched_lines: usize, pub matched_occurrences: usize, @@ -73,6 +77,20 @@ pub(crate) async fn remote_workspace_search_service( remote_workspace_search_service_for_path(root_path, preferred_connection_id).await } +/// flashgrep refuses to open a directory that is not a Git worktree with a HEAD commit. +/// That is a property of the workspace, not a failure of the index, so it is normalized into a +/// stable BitFun-owned sentence the UI can recognize instead of leaking the raw daemon error. +pub(crate) const NON_GIT_WORKSPACE_MESSAGE: &str = + "Workspace search requires a Git worktree with a HEAD commit"; + +fn repo_status_error_message(error: impl std::fmt::Display) -> String { + let message = error.to_string(); + if message.contains("requires a Git worktree with a HEAD commit") { + return NON_GIT_WORKSPACE_MESSAGE.to_string(); + } + format!("Failed to get search repository status: {message}") +} + async fn workspace_search_unavailable_message( state: &State<'_, AppState>, root_path: &str, @@ -169,8 +187,6 @@ pub(crate) fn build_content_search_request( use_regex, whole_word, multiline: false, - before_context: 0, - after_context: 0, max_results: Some(max_results), globs: Vec::new(), file_types: Vec::new(), @@ -226,7 +242,8 @@ pub(crate) fn search_metadata_from_content_result( SearchMetadataResponse { backend: result.backend, repo_phase: result.repo_status.phase, - rebuild_recommended: result.repo_status.rebuild_recommended, + base_advance_in_progress: result.repo_status.base_advance_in_progress, + workspace_probe_pending: result.repo_status.workspace_probe_pending, candidate_docs: result.candidate_docs, matched_lines: result.matched_lines, matched_occurrences: result.matched_occurrences, @@ -248,7 +265,7 @@ pub async fn search_get_repo_status( .get_index_status(&request.root_path) .await .map(|status| serde_json::to_value(status).unwrap_or_else(|_| serde_json::json!({}))) - .map_err(|error| format!("Failed to get search repository status: {}", error)); + .map_err(repo_status_error_message); } state @@ -256,7 +273,7 @@ pub async fn search_get_repo_status( .get_index_status(&request.root_path) .await .map(|status| serde_json::to_value(status).unwrap_or_else(|_| serde_json::json!({}))) - .map_err(|error| format!("Failed to get search repository status: {}", error)) + .map_err(repo_status_error_message) } #[tauri::command] @@ -310,3 +327,23 @@ pub async fn search_rebuild_index( .map(|task| serde_json::to_value(task).unwrap_or_else(|_| serde_json::json!({}))) .map_err(|error| format!("Failed to rebuild workspace index: {}", error)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn non_git_workspace_error_is_normalized() { + let raw = "protocol error: protocol error: indexed daemon workspace requires a Git worktree with a HEAD commit: /home"; + assert_eq!(repo_status_error_message(raw), NON_GIT_WORKSPACE_MESSAGE); + } + + #[test] + fn other_errors_keep_their_prefix() { + let raw = "SSH handshake timed out after 30 seconds"; + assert_eq!( + repo_status_error_message(raw), + "Failed to get search repository status: SSH handshake timed out after 30 seconds" + ); + } +} diff --git a/src/apps/desktop/src/api/workspace_activation.rs b/src/apps/desktop/src/api/workspace_activation.rs index 1fcff53acd..02e50e931c 100644 --- a/src/apps/desktop/src/api/workspace_activation.rs +++ b/src/apps/desktop/src/api/workspace_activation.rs @@ -1,5 +1,7 @@ use crate::api::app_state::AppState; -use bitfun_core::service::search::workspace_search_runtime_available; +use bitfun_core::service::search::{ + workspace_search_runtime_available, WorkspaceSearchAutoIndexPriority, +}; use bitfun_core::service::workspace::{WorkspaceInfo, WorkspaceKind}; use log::{debug, info, warn}; use std::path::{Path, PathBuf}; @@ -23,6 +25,62 @@ pub fn spawn_workspace_background_warmup(state: &AppState, workspace_info: Works }); } +pub fn spawn_restored_workspace_auto_index( + state: &AppState, + workspaces: Vec, + budget_roots: Vec, +) { + let workspace_path = state.workspace_path.clone(); + let workspace_search_service = state.workspace_search_service.clone(); + tokio::spawn(async move { + if !workspace_search_runtime_available().await { + return; + } + + let focused_path = workspace_path.read().await.clone(); + let protected_roots = focused_path + .as_ref() + .filter(|path| { + workspaces.iter().any(|workspace| { + workspace.workspace_kind != WorkspaceKind::Remote + && workspace.root_path == **path + }) + }) + .cloned() + .into_iter() + .collect(); + workspace_search_service + .enforce_index_disk_budget(budget_roots, protected_roots) + .await; + + if let Some(focused) = workspaces.iter().find(|workspace| { + workspace.workspace_kind != WorkspaceKind::Remote + && focused_path.as_ref() == Some(&workspace.root_path) + }) { + workspace_search_service + .schedule_auto_index( + &focused.root_path, + WorkspaceSearchAutoIndexPriority::Focused, + ) + .await; + } + + for workspace in workspaces { + if workspace.workspace_kind == WorkspaceKind::Remote + || focused_path.as_ref() == Some(&workspace.root_path) + { + continue; + } + workspace_search_service + .schedule_auto_index( + workspace.root_path, + WorkspaceSearchAutoIndexPriority::Background, + ) + .await; + } + }); +} + async fn warm_workspace_background_services( workspace_path: Arc>>, agent_registry: Arc, @@ -50,6 +108,16 @@ async fn warm_workspace_background_services( match workspace_search_service.open_repo(&target_path).await { Ok(_) => { let still_active = is_workspace_active(&workspace_path, &target_path).await; + workspace_search_service + .schedule_auto_index( + target_path.clone(), + if still_active { + WorkspaceSearchAutoIndexPriority::Focused + } else { + WorkspaceSearchAutoIndexPriority::Background + }, + ) + .await; if !still_active { workspace_search_service.schedule_repo_release(target_path.clone()); debug!( diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index bd8f3db5ba..f642bc6364 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -2645,6 +2645,10 @@ fn spawn_workspace_search_feature_listener(app_handle: tauri::AppHandle) { { match workspace_search_service.open_repo(¤t_workspace).await { Ok(_) => { + workspace_search_service.schedule_auto_index( + ¤t_workspace, + bitfun_core::service::search::WorkspaceSearchAutoIndexPriority::Focused, + ).await; log::info!( "Workspace search feature enabled; warmed current workspace: path={}", current_workspace.display() diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/glob_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/glob_tool.rs index 287c769057..443dba2303 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/glob_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/glob_tool.rs @@ -1,4 +1,5 @@ use crate::agentic::tools::framework::{Tool, ToolResult, ToolUseContext}; +use crate::agentic::tools::implementations::grep_tool::annotate_workspace_probe_pending; use crate::service::search::{ get_global_workspace_search_service, remote_workspace_search_service_for_path, workspace_search_feature_enabled, workspace_search_runtime_available, GlobSearchRequest, @@ -305,9 +306,13 @@ impl Tool for GlobTool { "total_matches": total_matches, "truncated": truncated, "repo_phase": glob_result.repo_status.phase, - "rebuild_recommended": glob_result.repo_status.rebuild_recommended + "base_advance_in_progress": glob_result.repo_status.base_advance_in_progress, + "workspace_probe_pending": glob_result.repo_status.workspace_probe_pending }), - result_for_assistant: Some(result_text), + result_for_assistant: Some(annotate_workspace_probe_pending( + result_text, + glob_result.repo_status.workspace_probe_pending, + )), image_attachments: None, }]) } @@ -453,9 +458,13 @@ impl Tool for GlobTool { "total_matches": total_matches, "truncated": truncated, "repo_phase": glob_result.repo_status.phase, - "rebuild_recommended": glob_result.repo_status.rebuild_recommended + "base_advance_in_progress": glob_result.repo_status.base_advance_in_progress, + "workspace_probe_pending": glob_result.repo_status.workspace_probe_pending }), - result_for_assistant: Some(result_text), + result_for_assistant: Some(annotate_workspace_probe_pending( + result_text, + glob_result.repo_status.workspace_probe_pending, + )), image_attachments: None, }]); } diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/grep_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/grep_tool.rs index bde735d576..5a7ccd24de 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/grep_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/grep_tool.rs @@ -20,6 +20,28 @@ use tool_runtime::search::grep_search::{ const DEFAULT_HEAD_LIMIT: usize = 250; +/// Prefixed to workspace-search output when the daemon's worktree view is behind. +/// +/// No search path waits for the daemon to reconcile the worktree: on a large repository that wait is +/// seconds, and it would land on whichever query happened to come first. The staleness is stated +/// instead. That keeps the failure mode legible — a caller that just edited a file can reconcile the +/// difference itself, but only if it is told the view may predate the edit. +pub(crate) const WORKSPACE_PROBE_PENDING_NOTE: &str = "Note: the workspace index is still folding in recent worktree changes, so these results describe the repository as of a moment ago. Very recent edits may be missing; re-run the search if a match you expect is absent."; + +/// Prepends [`WORKSPACE_PROBE_PENDING_NOTE`] to `body` when the daemon reported a pending probe. +pub(crate) fn annotate_workspace_probe_pending( + body: String, + workspace_probe_pending: bool, +) -> String { + if !workspace_probe_pending { + return body; + } + if body.is_empty() { + return WORKSPACE_PROBE_PENDING_NOTE.to_string(); + } + format!("{WORKSPACE_PROBE_PENDING_NOTE}\n\n{body}") +} + pub struct GrepTool; impl Default for GrepTool { @@ -262,6 +284,18 @@ impl GrepTool { Ok(options) } + /// Whether the caller asked for surrounding context lines (`-A` / `-B` / `-C` / `context`). + /// + /// The flashgrep daemon protocol has no context-line concept, so these requests must be + /// served by the ripgrep path instead of workspace search. + fn context_lines_requested(input: &Value) -> bool { + ["-A", "-B", "-C", "context"] + .iter() + .filter_map(|key| input.get(*key)) + .filter_map(|value| value.as_u64()) + .any(|lines| lines > 0) + } + fn build_workspace_search_request( &self, input: &Value, @@ -292,13 +326,6 @@ impl GrepTool { let offset = Self::resolve_offset(input); let head_limit = Self::resolve_head_limit(input); let max_results = Self::backend_max_results(input, offset, head_limit); - let before_context = input.get("-B").and_then(|v| v.as_u64()).unwrap_or(0) as usize; - let after_context = input.get("-A").and_then(|v| v.as_u64()).unwrap_or(0) as usize; - let shared_context = input - .get("context") - .or_else(|| input.get("-C")) - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; let globs = Self::parse_glob_patterns(input.get("glob").and_then(|v| v.as_str())); let file_types = input .get("type") @@ -322,16 +349,6 @@ impl GrepTool { .get("multiline") .and_then(|v| v.as_bool()) .unwrap_or(false), - before_context: if shared_context > 0 { - shared_context - } else { - before_context - }, - after_context: if shared_context > 0 { - shared_context - } else { - after_context - }, max_results, globs, file_types, @@ -410,24 +427,46 @@ impl GrepTool { } } +/// Renders one line per content match. +/// +/// Matches hydrated from disk carry their text. Transports that cannot read the +/// matched files (remote SSH) surface positions only, because the flashgrep +/// daemon never sends line text on the wire; those render as a bare +/// `path:line:` locator rather than being dropped, so the caller still learns +/// where the matches are. A match with neither text nor a line number carries no +/// usable information and is skipped. fn render_workspace_search_result_lines( results: &[crate::infrastructure::FileSearchResult], show_line_numbers: bool, ) -> Vec { - results - .iter() - .filter_map(|result| { - let content = result.matched_content.as_deref()?.trim_end(); - if show_line_numbers { - result - .line_number - .map(|line| format!("{}:{}:{}", result.path, line, content)) - .or_else(|| Some(format!("{}:{}", result.path, content))) - } else { - Some(format!("{}:{}", result.path, content)) + let mut lines: Vec = Vec::with_capacity(results.len()); + + for result in results { + let content = result + .matched_content + .as_deref() + .map(str::trim_end) + .filter(|content| !content.is_empty()); + + let rendered = match (content, result.line_number) { + (Some(content), Some(line)) if show_line_numbers => { + format!("{}:{}:{}", result.path, line, content) } - }) - .collect() + (Some(content), _) => format!("{}:{}", result.path, content), + (None, Some(line)) if show_line_numbers => format!("{}:{}:", result.path, line), + // Without line numbers a text-less match collapses to its path, so + // avoid repeating the same path once per match in the same file. + (None, Some(_)) => result.path.clone(), + (None, None) => continue, + }; + + if lines.last().is_some_and(|last| last == &rendered) { + continue; + } + lines.push(rendered); + } + + lines } fn render_workspace_search_content_lines( @@ -587,8 +626,12 @@ Usage: let focused_excluded_paths = crate::agentic::deep_review::scope::focused_review_excluded_changed_paths(context)?; + // The flashgrep daemon has no context-line support, so `-A`/`-B`/`-C` must go + // through the ripgrep path to produce surrounding lines. + let context_lines_requested = Self::context_lines_requested(input); + if resolved.uses_remote_workspace_backend() { - if workspace_search_feature_enabled().await { + if !context_lines_requested && workspace_search_feature_enabled().await { let remote_workspace_search_result = async { let (request, output_mode, show_line_numbers, offset, head_limit) = self.build_workspace_search_request(input, context)?; @@ -626,7 +669,7 @@ Usage: let workspace_search_elapsed_ms = search_started_at.elapsed().as_millis(); log::info!( - "Grep tool remote workspace-search result: pattern={}, path={}, output_mode={}, file_count={}, total_matches={}, backend={:?}, repo_phase={:?}, rebuild_recommended={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, workspace_search_ms={}", + "Grep tool remote workspace-search result: pattern={}, path={}, output_mode={}, file_count={}, total_matches={}, backend={:?}, repo_phase={:?}, base_advance_in_progress={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, workspace_search_ms={}", pattern, path, output_mode, @@ -634,7 +677,7 @@ Usage: total_matches, search_result.backend, search_result.repo_status.phase, - search_result.repo_status.rebuild_recommended, + search_result.repo_status.base_advance_in_progress, search_result.repo_status.dirty_files.modified, search_result.repo_status.dirty_files.deleted, search_result.repo_status.dirty_files.new, @@ -653,12 +696,16 @@ Usage: "total_matches": total_matches, "backend": search_result.backend, "repo_phase": search_result.repo_status.phase, - "rebuild_recommended": search_result.repo_status.rebuild_recommended, + "base_advance_in_progress": search_result.repo_status.base_advance_in_progress, + "workspace_probe_pending": search_result.repo_status.workspace_probe_pending, "applied_limit": head_limit, "applied_offset": if offset > 0 { Some(offset) } else { None:: }, "result": result_text, }), - result_for_assistant: Some(result_text), + result_for_assistant: Some(annotate_workspace_probe_pending( + result_text, + search_result.repo_status.workspace_probe_pending, + )), image_attachments: None, }]) } @@ -677,7 +724,10 @@ Usage: return self.call_remote(input, context).await; } - if focused_excluded_paths.is_none() && workspace_search_runtime_available().await { + if focused_excluded_paths.is_none() + && !context_lines_requested + && workspace_search_runtime_available().await + { if let Some(search_service) = get_global_workspace_search_service() { let (request, output_mode, show_line_numbers, offset, head_limit) = self.build_workspace_search_request(input, context)?; @@ -688,54 +738,68 @@ Usage: .map(|path| path.to_string_lossy().to_string()) .unwrap_or_else(|| request.repo_root.to_string_lossy().to_string()); let search_started_at = Instant::now(); - let search_result = search_service.search_content(request).await?; - let display_base = Self::display_base(context); - let (result_text, file_count, total_matches) = self.format_workspace_search_output( - &output_mode, - show_line_numbers, - offset, - head_limit, - &search_result, - display_base.as_deref(), - ); - let workspace_search_elapsed_ms = search_started_at.elapsed().as_millis(); - - log::info!( - "Grep tool workspace-search result: pattern={}, path={}, output_mode={}, file_count={}, total_matches={}, backend={:?}, repo_phase={:?}, rebuild_recommended={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, workspace_search_ms={}", - pattern, - path, - output_mode, - file_count, - total_matches, - search_result.backend, - search_result.repo_status.phase, - search_result.repo_status.rebuild_recommended, - search_result.repo_status.dirty_files.modified, - search_result.repo_status.dirty_files.deleted, - search_result.repo_status.dirty_files.new, - search_result.candidate_docs, - search_result.matched_lines, - search_result.matched_occurrences, - workspace_search_elapsed_ms, - ); + match search_service.search_content(request).await { + Ok(search_result) => { + let display_base = Self::display_base(context); + let (result_text, file_count, total_matches) = self + .format_workspace_search_output( + &output_mode, + show_line_numbers, + offset, + head_limit, + &search_result, + display_base.as_deref(), + ); + let workspace_search_elapsed_ms = search_started_at.elapsed().as_millis(); + + log::info!( + "Grep tool workspace-search result: pattern={}, path={}, output_mode={}, file_count={}, total_matches={}, backend={:?}, repo_phase={:?}, base_advance_in_progress={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, workspace_search_ms={}", + pattern, + path, + output_mode, + file_count, + total_matches, + search_result.backend, + search_result.repo_status.phase, + search_result.repo_status.base_advance_in_progress, + search_result.repo_status.dirty_files.modified, + search_result.repo_status.dirty_files.deleted, + search_result.repo_status.dirty_files.new, + search_result.candidate_docs, + search_result.matched_lines, + search_result.matched_occurrences, + workspace_search_elapsed_ms, + ); - return Ok(vec![ToolResult::Result { - data: json!({ - "pattern": pattern, - "path": path, - "output_mode": output_mode, - "file_count": file_count, - "total_matches": total_matches, - "backend": search_result.backend, - "repo_phase": search_result.repo_status.phase, - "rebuild_recommended": search_result.repo_status.rebuild_recommended, - "applied_limit": head_limit, - "applied_offset": if offset > 0 { Some(offset) } else { None:: }, - "result": result_text, - }), - result_for_assistant: Some(result_text), - image_attachments: None, - }]); + return Ok(vec![ToolResult::Result { + data: json!({ + "pattern": pattern, + "path": path, + "output_mode": output_mode, + "file_count": file_count, + "total_matches": total_matches, + "backend": search_result.backend, + "repo_phase": search_result.repo_status.phase, + "base_advance_in_progress": search_result.repo_status.base_advance_in_progress, + "workspace_probe_pending": search_result.repo_status.workspace_probe_pending, + "applied_limit": head_limit, + "applied_offset": if offset > 0 { Some(offset) } else { None:: }, + "result": result_text, + }), + result_for_assistant: Some(annotate_workspace_probe_pending( + result_text, + search_result.repo_status.workspace_probe_pending, + )), + image_attachments: None, + }]); + } + Err(error) => { + log::warn!( + "Grep tool workspace-search failed; falling back to local rg: {}", + error + ); + } + } } } @@ -802,6 +866,9 @@ Usage: result_text, applied_limit, applied_offset, + // Always false here: this call site supplies no cancellation token, so the search has + // no way to stop early. + cancelled: _, } = match search_result { Ok(Ok(result)) => result, Ok(Err(e)) => return Err(BitFunError::tool(e)), @@ -828,8 +895,9 @@ Usage: #[cfg(test)] mod tests { use super::{ - render_workspace_search_content_lines, render_workspace_search_result_lines, GrepTool, - DEFAULT_HEAD_LIMIT, + annotate_workspace_probe_pending, render_workspace_search_content_lines, + render_workspace_search_result_lines, GrepTool, DEFAULT_HEAD_LIMIT, + WORKSPACE_PROBE_PENDING_NOTE, }; use crate::infrastructure::{FileSearchOutcome, FileSearchResult, SearchMatchType}; use crate::service::search::{ @@ -856,6 +924,21 @@ mod tests { ); } + #[test] + fn context_lines_requested_detects_every_context_flag() { + assert!(!GrepTool::context_lines_requested(&json!({}))); + assert!(!GrepTool::context_lines_requested( + &json!({ "pattern": "foo", "-A": 0, "-B": 0, "-C": 0 }) + )); + + for key in ["-A", "-B", "-C", "context"] { + assert!( + GrepTool::context_lines_requested(&json!({ "pattern": "foo", key: 2 })), + "expected {key} to route the request to ripgrep" + ); + } + } + #[test] fn backend_max_results_only_uses_explicit_limit() { assert_eq!( @@ -999,6 +1082,12 @@ mod tests { .to_string(), phase: WorkspaceSearchRepoPhase::Ready, snapshot_key: None, + base_head_commit: None, + workspace_head_commit: None, + base_advance_in_progress: false, + base_advance_target_head: None, + base_delta_depth: 0, + base_compaction_recommended: false, last_probe_unix_secs: None, last_rebuild_unix_secs: None, dirty_files: crate::service::search::WorkspaceSearchDirtyFiles { @@ -1006,10 +1095,11 @@ mod tests { deleted: 0, new: 0, }, - rebuild_recommended: false, active_task_id: None, probe_healthy: true, + workspace_probe_pending: false, last_error: None, + last_maintenance_error: None, overlay: None, }, candidate_docs: 1, @@ -1072,6 +1162,12 @@ mod tests { .to_string(), phase: WorkspaceSearchRepoPhase::Ready, snapshot_key: None, + base_head_commit: None, + workspace_head_commit: None, + base_advance_in_progress: false, + base_advance_target_head: None, + base_delta_depth: 0, + base_compaction_recommended: false, last_probe_unix_secs: None, last_rebuild_unix_secs: None, dirty_files: crate::service::search::WorkspaceSearchDirtyFiles { @@ -1079,10 +1175,11 @@ mod tests { deleted: 0, new: 0, }, - rebuild_recommended: false, active_task_id: None, probe_healthy: true, + workspace_probe_pending: false, last_error: None, + last_maintenance_error: None, overlay: None, }, candidate_docs: 2, @@ -1120,4 +1217,78 @@ mod tests { assert_eq!(lines, vec!["/repo/src/main.rs:panic!(\"x\")"]); } + + #[test] + fn renders_locators_for_matches_without_line_text() { + let positions_only = |line: usize| FileSearchResult { + path: "/repo/src/main.rs".to_string(), + name: "main.rs".to_string(), + is_directory: false, + match_type: SearchMatchType::Content, + line_number: Some(line), + matched_content: None, + preview_before: None, + preview_inside: None, + preview_after: None, + }; + + let lines = + render_workspace_search_result_lines(&[positions_only(12), positions_only(73)], true); + assert_eq!( + lines, + vec!["/repo/src/main.rs:12:", "/repo/src/main.rs:73:"] + ); + + // Without line numbers there is nothing left but the path, so repeated + // matches in one file collapse to a single line. + let lines = + render_workspace_search_result_lines(&[positions_only(12), positions_only(73)], false); + assert_eq!(lines, vec!["/repo/src/main.rs"]); + } + + #[test] + fn skips_matches_without_text_or_line_number() { + let lines = render_workspace_search_result_lines( + &[FileSearchResult { + path: "/repo/src/main.rs".to_string(), + name: "main.rs".to_string(), + is_directory: false, + match_type: SearchMatchType::Content, + line_number: None, + matched_content: None, + preview_before: None, + preview_inside: None, + preview_after: None, + }], + true, + ); + + assert!(lines.is_empty()); + } + + #[test] + fn stale_workspace_view_is_stated_in_the_output_the_model_reads() { + // No search path waits for the daemon to reconcile, so the only thing that keeps a stale + // result from silently misleading the caller is saying so in the text it reads. + let annotated = annotate_workspace_probe_pending("src/lib.rs:1:hit".to_string(), true); + assert!(annotated.starts_with(WORKSPACE_PROBE_PENDING_NOTE)); + assert!(annotated.ends_with("src/lib.rs:1:hit")); + } + + #[test] + fn a_current_workspace_view_adds_nothing_to_the_output() { + // The pending case is the exception; the common case must stay byte-identical so the note + // never becomes background noise the model learns to skip. + let body = "src/lib.rs:1:hit".to_string(); + assert_eq!(annotate_workspace_probe_pending(body.clone(), false), body); + } + + #[test] + fn a_stale_empty_result_still_says_why_it_may_be_empty() { + // "No matches" plus a stale index is exactly the case where the caller needs the note most. + assert_eq!( + annotate_workspace_probe_pending(String::new(), true), + WORKSPACE_PROBE_PENDING_NOTE + ); + } } diff --git a/src/crates/assembly/core/src/service/mod.rs b/src/crates/assembly/core/src/service/mod.rs index 5f03f47eb9..6bd6c57d89 100644 --- a/src/crates/assembly/core/src/service/mod.rs +++ b/src/crates/assembly/core/src/service/mod.rs @@ -119,7 +119,8 @@ pub use runtime::{ResolvedCommand, RuntimeCommandCapability, RuntimeManager, Run pub use search::{ get_global_workspace_search_service, set_global_workspace_search_service, ContentSearchRequest, ContentSearchResult, GlobSearchRequest, GlobSearchResult, IndexTaskHandle, - WorkspaceIndexStatus, WorkspaceSearchBackend, WorkspaceSearchContextLine, + WorkspaceIndexStatus, WorkspaceSearchAutoIndexDecision, WorkspaceSearchAutoIndexPriority, + WorkspaceSearchAutoIndexStatus, WorkspaceSearchBackend, WorkspaceSearchContextLine, WorkspaceSearchDirtyFiles, WorkspaceSearchFileCount, WorkspaceSearchHit, WorkspaceSearchLine, WorkspaceSearchMatch, WorkspaceSearchMatchLocation, WorkspaceSearchOverlayStatus, WorkspaceSearchRepoPhase, WorkspaceSearchRepoStatus, WorkspaceSearchService, diff --git a/src/crates/assembly/core/src/service/search/mod.rs b/src/crates/assembly/core/src/service/search/mod.rs index 933534ca1b..9cf11b5a65 100644 --- a/src/crates/assembly/core/src/service/search/mod.rs +++ b/src/crates/assembly/core/src/service/search/mod.rs @@ -6,7 +6,8 @@ pub mod service; pub use bitfun_services_integrations::workspace_search::{ ContentSearchOutputMode, ContentSearchRequest, ContentSearchResult, GlobSearchRequest, - GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchBackend, + GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchAutoIndexDecision, + WorkspaceSearchAutoIndexPriority, WorkspaceSearchAutoIndexStatus, WorkspaceSearchBackend, WorkspaceSearchContextLine, WorkspaceSearchDirtyFiles, WorkspaceSearchFileCount, WorkspaceSearchHit, WorkspaceSearchLine, WorkspaceSearchMatch, WorkspaceSearchMatchLocation, WorkspaceSearchOverlayStatus, WorkspaceSearchRepoPhase, WorkspaceSearchRepoStatus, diff --git a/src/crates/assembly/core/src/service/search/service.rs b/src/crates/assembly/core/src/service/search/service.rs index b3cd84097c..1aa700523c 100644 --- a/src/crates/assembly/core/src/service/search/service.rs +++ b/src/crates/assembly/core/src/service/search/service.rs @@ -86,6 +86,28 @@ impl WorkspaceSearchService { self.inner.schedule_repo_release(repo_root); } + pub async fn schedule_auto_index( + self: &Arc, + repo_root: impl AsRef, + priority: owner::WorkspaceSearchAutoIndexPriority, + ) { + self.inner.schedule_auto_index(repo_root, priority).await; + } + + pub async fn enforce_index_disk_budget( + &self, + workspace_roots: Vec, + protected_roots: Vec, + ) { + self.inner + .enforce_index_disk_budget(workspace_roots, protected_roots) + .await; + } + + pub async fn remove_workspace_index(&self, repo_root: impl AsRef) { + self.inner.remove_workspace_index(repo_root).await; + } + pub async fn shutdown_all_daemons(&self) { self.inner.shutdown_all_daemons().await; } diff --git a/src/crates/assembly/core/src/service/workspace/service.rs b/src/crates/assembly/core/src/service/workspace/service.rs index 7617934ab3..7aab36468f 100644 --- a/src/crates/assembly/core/src/service/workspace/service.rs +++ b/src/crates/assembly/core/src/service/workspace/service.rs @@ -1076,12 +1076,25 @@ impl WorkspaceService { /// Removes a workspace. pub async fn remove_workspace(&self, workspace_id: &str) -> BitFunResult<()> { - let result = { + let (removed_workspace, result) = { let mut manager = self.manager.write().await; - manager.remove_workspace(workspace_id) + let workspace = manager.get_workspace(workspace_id).cloned(); + let result = manager.remove_workspace(workspace_id); + (workspace, result) }; if result.is_ok() { + if let Some(workspace) = removed_workspace { + if workspace.workspace_kind != WorkspaceKind::Remote { + if let Some(search_service) = + crate::service::search::get_global_workspace_search_service() + { + search_service + .remove_workspace_index(&workspace.root_path) + .await; + } + } + } if let Err(e) = self.save_workspace_data().await { warn!("Failed to save workspace data after removal: {}", e); } diff --git a/src/crates/execution/tool-execution/src/search/grep_search.rs b/src/crates/execution/tool-execution/src/search/grep_search.rs index a4d6449bc8..f9b8e09775 100644 --- a/src/crates/execution/tool-execution/src/search/grep_search.rs +++ b/src/crates/execution/tool-execution/src/search/grep_search.rs @@ -7,10 +7,10 @@ use std::sync::{Arc, Mutex}; use std::time::SystemTime; use globset::{GlobBuilder, GlobMatcher}; -use grep_regex::RegexMatcherBuilder; +use grep_regex::{RegexMatcher, RegexMatcherBuilder}; use grep_searcher::{Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch}; use ignore::types::TypesBuilder; -use ignore::WalkBuilder; +use ignore::{DirEntry, WalkBuilder, WalkState}; const MAX_DISPLAY_COLUMNS: usize = 500; const VCS_DIRECTORIES_TO_EXCLUDE: &[&str] = &[".git", ".svn", ".hg", ".bzr", ".jj", ".sl"]; @@ -244,6 +244,37 @@ impl Sink for GrepSink { /// Progress report callback type pub type ProgressCallback = Arc; +/// Cooperative cancellation for an in-flight [`grep_search`]. +/// +/// Cheap to clone and share: the walker threads, the per-worker searchers and the reducer all hold +/// the same flag. Cancelling is one-way — a cancelled search never becomes runnable again, so a +/// caller that wants to retry builds a fresh token. +/// +/// The contract is *cooperative*, not preemptive: a cancel is observed between filesystem entries, +/// so a single very large file still finishes being searched. What it bounds is the walk, which is +/// where the wall-clock actually goes on a large repository. +#[derive(Debug, Clone, Default)] +pub struct SearchCancellation { + flag: std::sync::Arc, +} + +impl SearchCancellation { + pub fn new() -> Self { + Self::default() + } + + /// Ask the search to stop. Idempotent, and safe to call from any thread. + pub fn cancel(&self) { + // Relaxed is enough: the flag guards no other data, and every reader is a plain poll whose + // only requirement is that the store eventually becomes visible. + self.flag.store(true, std::sync::atomic::Ordering::Relaxed); + } + + pub fn is_cancelled(&self) -> bool { + self.flag.load(std::sync::atomic::Ordering::Relaxed) + } +} + /// grep search options #[derive(Debug, Clone)] pub struct GrepOptions { @@ -279,6 +310,8 @@ pub struct GrepOptions { pub excluded_paths: Vec, /// Reject linked file entries when the caller requires workspace identity. pub reject_linked_files: bool, + /// Cooperative cancellation. `None` means the search cannot be cancelled. + pub cancellation: Option, } impl Default for GrepOptions { @@ -300,6 +333,7 @@ impl Default for GrepOptions { display_base: None, excluded_paths: Vec::new(), reject_linked_files: false, + cancellation: None, } } } @@ -450,6 +484,12 @@ impl GrepOptions { self } + /// Attach a cancellation token so the caller can stop this search early. + pub fn cancellation(mut self, value: SearchCancellation) -> Self { + self.cancellation = Some(value); + self + } + /// Set whether to enable multiline mode pub fn multiline(mut self, value: bool) -> Self { self.multiline = value; @@ -552,6 +592,111 @@ pub struct GrepSearchResult { pub result_text: String, pub applied_limit: Option, pub applied_offset: Option, + /// The search stopped early because its [`SearchCancellation`] fired. Everything already + /// aggregated is still returned — it is a correct result over the subset of the tree that was + /// walked, not over the whole tree — so callers that care about completeness must check this. + pub cancelled: bool, +} + +#[derive(Clone)] +struct GrepWorkerConfig { + output_mode: OutputMode, + show_line_numbers: bool, + before_context: usize, + after_context: usize, + display_base: Option, + globs: Vec, + excluded_paths: Vec, + reject_linked_files: bool, +} + +struct GrepFileResult { + path: PathBuf, + file_matches: usize, + output_lines: Vec, + modified_time: SystemTime, +} + +enum GrepWorkerEvent { + Processed(Option), + Error(String), +} + +fn build_grep_searcher(before_context: usize, after_context: usize, multiline: bool) -> Searcher { + let mut builder = SearcherBuilder::new(); + builder + .line_number(true) + .before_context(before_context) + .after_context(after_context); + if multiline { + builder.multi_line(true); + } + builder.build() +} + +fn search_entry( + entry: Result, + searcher: &mut Searcher, + matcher: &RegexMatcher, + config: &GrepWorkerConfig, +) -> GrepWorkerEvent { + let entry = match entry { + Ok(entry) => entry, + Err(error) => return GrepWorkerEvent::Error(format!("Error walking files: {error}")), + }; + let path = entry.path(); + + let entry_file_type = entry.file_type(); + let is_file = entry_file_type + .is_some_and(|file_type| file_type.is_file() || (file_type.is_symlink() && path.is_file())); + if !is_file { + return GrepWorkerEvent::Processed(None); + } + + let path_is_symlink = entry_file_type.is_some_and(|file_type| file_type.is_symlink()); + let path_has_multiple_hard_links = + config.reject_linked_files && crate::fs::path_has_multiple_hard_links(path).unwrap_or(true); + if config.reject_linked_files && (path_is_symlink || path_has_multiple_hard_links) { + return GrepWorkerEvent::Processed(None); + } + if config + .excluded_paths + .iter() + .any(|excluded| paths_equal_for_exclusion(path, excluded)) + || is_vcs_path(path) + || (!config.globs.is_empty() && !config.globs.iter().any(|glob| glob.is_match(path))) + { + return GrepWorkerEvent::Processed(None); + } + + let sink = GrepSink::new( + config.output_mode, + config.show_line_numbers, + config.before_context, + config.after_context, + None, + path.to_path_buf(), + config.display_base.clone(), + ); + if let Err(error) = searcher.search_path(matcher, path, sink.clone()) { + return GrepWorkerEvent::Error(format!("Error searching file {}: {error}", path.display())); + } + + let file_matches = sink.get_match_count(); + if file_matches == 0 { + return GrepWorkerEvent::Processed(None); + } + let output_lines = if config.output_mode == OutputMode::Content { + sink.take_output_lines() + } else { + Vec::new() + }; + GrepWorkerEvent::Processed(Some(GrepFileResult { + path: path.to_path_buf(), + file_matches, + output_lines, + modified_time: modified_time(path), + })) } fn is_vcs_path(path: &Path) -> bool { @@ -634,6 +779,23 @@ pub fn grep_search( return Err(format!("Search path '{}' does not exist", search_path)); } + let cancellation = options.cancellation.clone(); + let is_cancelled = move || { + cancellation + .as_ref() + .is_some_and(SearchCancellation::is_cancelled) + }; + if is_cancelled() { + return Ok(GrepSearchResult { + file_count: 0, + total_matches: 0, + result_text: String::new(), + applied_limit: None, + applied_offset: None, + cancelled: true, + }); + } + let before_context = options .before_context .unwrap_or(options.context.unwrap_or(0)); @@ -658,19 +820,6 @@ pub fn grep_search( .build(pattern) .map_err(|e| format!("Invalid regex pattern: {}", e))?; - // Build searcher - let mut searcher_builder = SearcherBuilder::new(); - searcher_builder - .line_number(true) - .before_context(before_context) - .after_context(after_context); - - if multiline { - searcher_builder.multi_line(true); - } - - let mut searcher = searcher_builder.build(); - // Build walker let mut walk_builder = WalkBuilder::new(search_path); walk_builder @@ -725,8 +874,6 @@ pub fn grep_search( } } - let walker = walk_builder.build(); - // Pre-build glob matcher let glob_matchers = options .globs @@ -739,137 +886,134 @@ pub fn grep_search( }) .collect::, String>>()?; + let worker_config = GrepWorkerConfig { + output_mode, + show_line_numbers, + before_context, + after_context, + display_base, + globs: glob_matchers, + excluded_paths: options.excluded_paths.clone(), + reject_linked_files: options.reject_linked_files, + }; + let worker_count = + std::thread::available_parallelism().map_or(1, |parallelism| parallelism.get().min(8)); + walk_builder.threads(worker_count); + let walker = walk_builder.build_parallel(); + // Collect all results - let mut content_lines: Vec = - Vec::with_capacity(head_limit.map_or(256, |limit| limit.min(4096))); let mut total_matches = 0; let mut file_count = 0; - let mut file_match_counts: Vec<(String, usize)> = Vec::new(); - let mut matched_files_with_mtime: Vec<(String, SystemTime)> = Vec::new(); + let (event_sender, event_receiver) = std::sync::mpsc::sync_channel(worker_count * 4); + let worker_matcher = matcher.clone(); + let reducer_config = worker_config.clone(); + let walker_cancellation = options.cancellation.clone(); + let walker_thread = std::thread::spawn(move || { + let sender = event_sender; + walker.run(move || { + let sender = sender.clone(); + let matcher = worker_matcher.clone(); + let config = worker_config.clone(); + let cancellation = walker_cancellation.clone(); + let mut searcher = + build_grep_searcher(config.before_context, config.after_context, multiline); + Box::new(move |entry| { + // Checked before the search, not after: the point of cancelling is to stop paying + // for work, and the per-entry search is the expensive half. + if cancellation + .as_ref() + .is_some_and(SearchCancellation::is_cancelled) + { + return WalkState::Quit; + } + let event = search_entry(entry, &mut searcher, &matcher, &config); + if sender.send(event).is_err() { + WalkState::Quit + } else { + WalkState::Continue + } + }) + }); + }); // Progress tracking let mut files_processed = 0; let mut last_progress_time = std::time::Instant::now(); let progress_interval_millis = progress_interval_millis.unwrap_or(500); - - // Traverse files and search - for result in walker { - match result { - Ok(entry) => { - let path = entry.path(); - + let mut file_results = Vec::new(); + + let mut cancelled = false; + for event in event_receiver { + if is_cancelled() { + // Stop consuming and let the receiver drop. Every worker's next `send` then fails, + // which is what unwinds the walk without needing a second signalling path. + cancelled = true; + break; + } + match event { + GrepWorkerEvent::Processed(result) => { files_processed += 1; - - if last_progress_time.elapsed().as_millis() >= progress_interval_millis { - info!( - "Search progress: processed {} files, found {} matching files, total {} matches", - files_processed, file_count, total_matches - ); - - if let Some(ref callback) = progress_callback { - callback(files_processed, file_count, total_matches); - } - - last_progress_time = std::time::Instant::now(); - } - - // Check if it's a file. Use the walker-provided file type - // (free) and only fall back to a stat for symlinks. - let entry_file_type = entry.file_type(); - let is_file = entry_file_type.is_some_and(|file_type| { - file_type.is_file() || (file_type.is_symlink() && path.is_file()) - }); - if !is_file { - continue; - } - - // Focused Review supplies exclusions. In that mode, linked - // file entries are never valid unchanged dependencies because - // their target identity can escape or alias the assigned scope. - let path_is_symlink = entry - .file_type() - .is_some_and(|file_type| file_type.is_symlink()); - let path_has_multiple_hard_links = options.reject_linked_files - && crate::fs::path_has_multiple_hard_links(path).unwrap_or(true); - if options.reject_linked_files && (path_is_symlink || path_has_multiple_hard_links) - { - continue; - } - - if options - .excluded_paths - .iter() - .any(|excluded| paths_equal_for_exclusion(path, excluded)) - { - continue; + if let Some(result) = result { + file_count += 1; + total_matches += result.file_matches; + file_results.push(result); } + } + GrepWorkerEvent::Error(error) => warn!("{}", error), + } - if is_vcs_path(path) { - continue; - } + if last_progress_time.elapsed().as_millis() >= progress_interval_millis { + info!( + "Search progress: processed {} files, found {} matching files, total {} matches", + files_processed, file_count, total_matches + ); - if !glob_matchers.is_empty() - && !glob_matchers.iter().any(|matcher| matcher.is_match(path)) - { - continue; - } + if let Some(ref callback) = progress_callback { + callback(files_processed, file_count, total_matches); + } - let sink = GrepSink::new( - output_mode, - show_line_numbers, - before_context, - after_context, - None, - path.to_path_buf(), - display_base.clone(), - ); - - // Execute search - if let Err(e) = searcher.search_path(&matcher, path, sink.clone()) { - warn!("Error searching file {}: {}", path.display(), e); - continue; - } + last_progress_time = std::time::Instant::now(); + } + } + // The receiver was moved into the `for` loop above and is dropped as that loop exits, break + // included — so by the time we join, the workers' sends are already failing. + if walker_thread.join().is_err() { + warn!("Parallel search walker thread panicked"); + } + // The walk can also observe the cancel first and quit on its own, which closes the channel and + // ends the loop above without setting the flag there. + let cancelled = cancelled || is_cancelled(); - let file_matches = sink.get_match_count(); - if file_matches > 0 { - file_count += 1; - total_matches += file_matches; - match output_mode { - OutputMode::Content => { - for line in sink.take_output_lines() { - // In multi-line mode a single sink write can - // span several physical lines; keep one entry - // per physical line so head_limit/offset still - // paginate by line. - if line.contains('\n') { - content_lines.extend( - line.lines() - .filter(|part| !part.is_empty()) - .map(str::to_string), - ); - } else if !line.is_empty() { - content_lines.push(line); - } - } - } - OutputMode::FilesWithMatches => { - matched_files_with_mtime.push(( - relativize_display_path(path, display_base.as_deref()), - modified_time(path), - )); - } - OutputMode::Count => { - file_match_counts.push(( - relativize_display_path(path, display_base.as_deref()), - file_matches, - )); - } + // Worker completion order is nondeterministic. Stable path order keeps Content and Count + // output reproducible; FilesWithMatches applies its existing mtime ordering below. + file_results.sort_by(|left, right| left.path.cmp(&right.path)); + let mut content_lines: Vec = + Vec::with_capacity(head_limit.map_or(256, |limit| limit.min(4096))); + let mut file_match_counts: Vec<(String, usize)> = Vec::new(); + let mut matched_files_with_mtime: Vec<(String, SystemTime)> = Vec::new(); + for result in file_results { + let display_path = + relativize_display_path(&result.path, reducer_config.display_base.as_deref()); + match output_mode { + OutputMode::Content => { + for line in result.output_lines { + // In multi-line mode a single sink write can span several physical lines; + // keep one entry per physical line so head_limit/offset paginate by line. + if line.contains('\n') { + content_lines.extend( + line.lines() + .filter(|part| !part.is_empty()) + .map(str::to_string), + ); + } else if !line.is_empty() { + content_lines.push(line); } } } - Err(e) => { - warn!("Error walking files: {}", e); + OutputMode::FilesWithMatches => { + matched_files_with_mtime.push((display_path, result.modified_time)); } + OutputMode::Count => file_match_counts.push((display_path, result.file_matches)), } } @@ -888,6 +1032,7 @@ pub fn grep_search( result_text: lines.join("\n"), applied_limit, applied_offset, + cancelled, }); } } @@ -910,6 +1055,7 @@ pub fn grep_search( result_text: matches.join("\n").trim_end_matches('\n').to_string(), applied_limit, applied_offset, + cancelled, }); } } @@ -938,6 +1084,7 @@ pub fn grep_search( .to_string(), applied_limit, applied_offset, + cancelled, }); } } @@ -949,6 +1096,7 @@ pub fn grep_search( result_text: result_text.trim_end_matches('\n').to_string(), applied_limit: None, applied_offset: if offset > 0 { Some(offset) } else { None }, + cancelled, }) } @@ -973,7 +1121,9 @@ fn paths_equal_for_exclusion(path: &Path, excluded: &str) -> bool { #[cfg(test)] mod tests { - use super::{grep_search, paths_equal_for_exclusion, GrepOptions, OutputMode}; + use super::{ + grep_search, paths_equal_for_exclusion, GrepOptions, OutputMode, SearchCancellation, + }; use std::fs; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -999,6 +1149,95 @@ mod tests { std::os::windows::fs::symlink_file(target, alias).is_ok() } + #[test] + fn a_search_that_was_never_cancelled_reports_so() { + let root = make_temp_dir("cancel-none"); + fs::write(root.join("a.txt"), "needle\n").unwrap(); + + let result = grep_search( + GrepOptions::new("needle", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content), + None, + None, + ) + .unwrap(); + + assert!(!result.cancelled); + assert_eq!(result.file_count, 1); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn cancelling_before_the_walk_starts_returns_an_empty_cancelled_result() { + let root = make_temp_dir("cancel-upfront"); + fs::write(root.join("a.txt"), "needle\n").unwrap(); + + let cancellation = SearchCancellation::new(); + cancellation.cancel(); + + let result = grep_search( + GrepOptions::new("needle", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content) + .cancellation(cancellation), + None, + None, + ) + .unwrap(); + + assert!(result.cancelled); + assert_eq!(result.file_count, 0); + assert_eq!(result.total_matches, 0); + assert!(result.result_text.is_empty()); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn cancelling_mid_walk_keeps_what_was_already_found() { + // Every file matches, so an uncancelled run would report all 200. The progress interval is + // zeroed so the callback fires on every event; we cancel once five matches are in, which + // puts the cancel mid-walk deterministically instead of racing a timer, and leaves a known + // non-empty partial result to assert on. + let root = make_temp_dir("cancel-midwalk"); + for index in 0..200 { + fs::write(root.join(format!("file-{index:03}.txt")), "needle\n").unwrap(); + } + + let cancellation = SearchCancellation::new(); + let from_callback = cancellation.clone(); + let callback: super::ProgressCallback = + std::sync::Arc::new(move |_files_processed, file_count, _total_matches| { + if file_count >= 5 { + from_callback.cancel(); + } + }); + + let result = grep_search( + GrepOptions::new("needle", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content) + .cancellation(cancellation), + Some(callback), + Some(0), + ) + .unwrap(); + + assert!(result.cancelled); + // Partial, not empty and not complete: the point of the contract is that a cancelled search + // still hands back the work it had already paid for. + assert!( + result.file_count < 200, + "expected the walk to stop early, saw {} files", + result.file_count + ); + // The cancel is raised from inside the callback once five matches are counted, so a + // correct implementation never comes back with fewer than that. + assert!(result.file_count >= 5, "saw {} files", result.file_count); + assert_eq!(result.result_text.lines().count(), result.file_count); + + let _ = fs::remove_dir_all(root); + } + #[test] fn truncates_very_long_output_lines() { let root = make_temp_dir("truncate"); @@ -1161,4 +1400,137 @@ mod tests { let _ = fs::remove_dir_all(root); let _ = fs::remove_dir_all(outside); } + + #[test] + fn parallel_output_modes_keep_stable_order_and_counts() { + let root = make_temp_dir("parallel-output"); + fs::write(root.join("z-last.txt"), "parallel-token\n").unwrap(); + fs::write(root.join("a-first.txt"), "parallel-token\n").unwrap(); + let display_base = root.to_string_lossy().to_string(); + + let content = grep_search( + GrepOptions::new("parallel-token", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content) + .display_base(display_base.clone()), + None, + None, + ) + .unwrap(); + assert_eq!( + content.result_text, + "a-first.txt:1:parallel-token\nz-last.txt:1:parallel-token" + ); + + let count = grep_search( + GrepOptions::new("parallel-token", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Count) + .display_base(display_base), + None, + None, + ) + .unwrap(); + assert_eq!(count.file_count, 2); + assert_eq!(count.total_matches, 2); + assert_eq!( + count.result_text, + "Total 2 matches in 2 files:\na-first.txt:1\nz-last.txt:1" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn parallel_content_order_is_stable_across_repeated_runs() { + // Two files can agree by luck; sixty spread over eight workers cannot. Names are written in + // an order unrelated to their sort order so a reducer that just preserved arrival order + // would produce something different from the sorted answer. + let root = make_temp_dir("parallel-stable-order"); + for index in (0..60).rev() { + fs::write( + root.join(format!("file-{index:02}.txt")), + "parallel-token\n", + ) + .unwrap(); + } + let display_base = root.to_string_lossy().to_string(); + + let run = || { + grep_search( + GrepOptions::new("parallel-token", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content) + .display_base(display_base.clone()), + None, + None, + ) + .unwrap() + .result_text + }; + + let expected: Vec = (0..60) + .map(|index| format!("file-{index:02}.txt:1:parallel-token")) + .collect(); + for _ in 0..5 { + assert_eq!(run(), expected.join("\n")); + } + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn pagination_slices_the_stable_parallel_order() { + // Pagination is only meaningful if the underlying order is fixed: windows taken with + // different offsets have to tile the full result exactly, with no gaps and no repeats. + let root = make_temp_dir("parallel-pagination"); + for index in 0..60 { + fs::write( + root.join(format!("file-{index:02}.txt")), + "parallel-token\n", + ) + .unwrap(); + } + let display_base = root.to_string_lossy().to_string(); + + let page = |offset: usize, limit: usize| { + grep_search( + GrepOptions::new("parallel-token", root.to_string_lossy().to_string()) + .output_mode(OutputMode::Content) + .display_base(display_base.clone()) + .offset(offset) + .head_limit(limit), + None, + None, + ) + .unwrap() + }; + + let full = page(0, 60); + let full_lines: Vec<&str> = full.result_text.lines().collect(); + assert_eq!(full_lines.len(), 60); + + let mut tiled = Vec::new(); + for offset in (0..60).step_by(7) { + let window = page(offset, 7); + assert_eq!(window.applied_offset, Some(offset).filter(|it| *it > 0)); + tiled.extend( + window + .result_text + .lines() + .map(str::to_string) + .collect::>(), + ); + } + assert_eq!(tiled, full_lines); + + // Past the end is an empty page, not an error and not a wrapped-around one. Note that the + // rendered text is the same "No matches found" string an genuinely empty search produces — + // callers that need to tell the two apart have to look at `total_matches`, which still + // reports the full unpaginated count. + let past_end = page(60, 7); + assert_eq!( + past_end.result_text, + "No matches found for pattern 'parallel-token'" + ); + assert_eq!(past_end.total_matches, 60); + + let _ = fs::remove_dir_all(root); + } } diff --git a/src/crates/execution/tool-execution/src/search/mod.rs b/src/crates/execution/tool-execution/src/search/mod.rs index c2f684478e..e52e9c22a1 100644 --- a/src/crates/execution/tool-execution/src/search/mod.rs +++ b/src/crates/execution/tool-execution/src/search/mod.rs @@ -9,5 +9,5 @@ pub use glob_search::{ pub use grep_search::{ apply_offset_and_limit, build_remote_grep_command, count_remote_grep_matches, grep_search, relativize_result_text, render_remote_grep_result_text, GrepOptions, OutputMode, - ProgressCallback, RemoteGrepCommandRequest, + ProgressCallback, RemoteGrepCommandRequest, SearchCancellation, }; diff --git a/src/crates/services/services-core/src/filesystem/content_preview.rs b/src/crates/services/services-core/src/filesystem/content_preview.rs new file mode 100644 index 0000000000..470bcf6461 --- /dev/null +++ b/src/crates/services/services-core/src/filesystem/content_preview.rs @@ -0,0 +1,218 @@ +//! Content-match preview primitives shared by every content search backend. +//! +//! The local walker (`tree.rs`) and the flashgrep-backed workspace search must +//! render the same preview shape, otherwise the same match looks different +//! depending on whether the workspace happens to be indexed. Both the pattern → +//! regex translation and the truncation budget therefore live here instead of +//! being duplicated per backend. + +use super::error::{FileSystemError, FileSystemResult}; +use regex::{Regex, RegexBuilder}; + +/// Total preview budget across the before/inside/after segments. +const MAX_PREVIEW_CHARS: usize = 250; +/// Budget for the text preceding the match; the rest is spent on the match and +/// its trailing context so that the match itself stays visible on screen. +const MAX_PREVIEW_BEFORE_CHARS: usize = 26; + +/// Compiles a user-facing search pattern into the matcher used for previews. +/// +/// The literal/whole-word translation is part of the contract: callers pass the +/// raw pattern plus the same flags they hand to their search backend, so the +/// preview highlights exactly what the backend matched. +pub fn compile_content_search_regex( + pattern: &str, + case_sensitive: bool, + use_regex: bool, + whole_word: bool, +) -> Result { + let search_pattern = if use_regex { + pattern.to_string() + } else if whole_word { + format!(r"\b{}\b", regex::escape(pattern)) + } else { + regex::escape(pattern) + }; + + RegexBuilder::new(&search_pattern) + .case_insensitive(!case_sensitive) + .build() +} + +/// Splits a matched line into `(before, inside, after)` preview segments. +/// +/// Returns `(None, None, None)` when the matcher does not match the line, which +/// happens whenever the caller's matcher is not the one that produced the line +/// (for example a daemon-side regex dialect the client cannot reproduce). +pub fn build_content_match_preview( + line: &str, + matcher: &Regex, +) -> (Option, Option, Option) { + let Some(found_match) = matcher.find(line) else { + return (None, None, None); + }; + + let full_before = &line[..found_match.start()]; + let before = left_truncate_with_ellipsis(full_before, MAX_PREVIEW_BEFORE_CHARS); + + let mut chars_remaining = MAX_PREVIEW_CHARS.saturating_sub(before.chars().count()); + let mut inside = take_first_chars(found_match.as_str(), chars_remaining); + chars_remaining = chars_remaining.saturating_sub(inside.chars().count()); + let after = take_first_chars(&line[found_match.end()..], chars_remaining); + + if inside.is_empty() { + inside = found_match.as_str().to_string(); + } + + (Some(before), Some(inside), Some(after)) +} + +/// A compiled preview matcher. +/// +/// Backends that receive matched lines from somewhere else (a search daemon, a +/// remote host) get the line text without any match offsets, so they recompute +/// the highlight locally. This wrapper keeps `regex` out of their dependency +/// surface: they hand over the same pattern and flags they searched with, then +/// ask for a preview per line. +pub struct ContentMatchPreviewBuilder { + matcher: Regex, +} + +impl ContentMatchPreviewBuilder { + pub fn new( + pattern: &str, + case_sensitive: bool, + use_regex: bool, + whole_word: bool, + ) -> FileSystemResult { + let matcher = compile_content_search_regex(pattern, case_sensitive, use_regex, whole_word) + .map_err(|error| { + FileSystemError::service(format!("Invalid regex pattern: {}", error)) + })?; + Ok(Self { matcher }) + } + + /// Splits `line` into `(before, inside, after)` preview segments. + pub fn preview(&self, line: &str) -> (Option, Option, Option) { + build_content_match_preview(line, &self.matcher) + } +} + +fn take_first_chars(text: &str, max_chars: usize) -> String { + if max_chars == 0 { + return String::new(); + } + + let mut end_index = text.len(); + for (char_count, (byte_index, _)) in text.char_indices().enumerate() { + if char_count == max_chars { + end_index = byte_index; + break; + } + } + + text[..end_index].to_string() +} + +fn left_truncate_with_ellipsis(text: &str, max_chars: usize) -> String { + let total_chars = text.chars().count(); + if total_chars <= max_chars { + return text.to_string(); + } + + if max_chars <= 1 { + return "\u{2026}".to_string(); + } + + let keep_chars = max_chars - 1; + let start_index = text + .char_indices() + .nth(total_chars.saturating_sub(keep_chars)) + .map(|(index, _)| index) + .unwrap_or(0); + + format!("\u{2026}{}", &text[start_index..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn literal_pattern_is_escaped() { + let matcher = compile_content_search_regex("a.c", true, false, false).expect("regex"); + assert!(matcher.is_match("a.c")); + assert!(!matcher.is_match("abc")); + } + + #[test] + fn whole_word_literal_requires_boundaries() { + let matcher = compile_content_search_regex("cat", true, false, true).expect("regex"); + assert!(matcher.is_match("a cat here")); + assert!(!matcher.is_match("concatenate")); + } + + #[test] + fn case_insensitive_is_the_default_when_not_case_sensitive() { + let matcher = compile_content_search_regex("Cat", false, false, false).expect("regex"); + assert!(matcher.is_match("CAT")); + } + + #[test] + fn preview_splits_line_around_the_match() { + let matcher = compile_content_search_regex("needle", true, false, false).expect("regex"); + let (before, inside, after) = build_content_match_preview("a needle here", &matcher); + assert_eq!(before.as_deref(), Some("a ")); + assert_eq!(inside.as_deref(), Some("needle")); + assert_eq!(after.as_deref(), Some(" here")); + } + + #[test] + fn preview_is_empty_when_the_matcher_does_not_match() { + let matcher = compile_content_search_regex("needle", true, false, false).expect("regex"); + assert_eq!( + build_content_match_preview("nothing here", &matcher), + (None, None, None) + ); + } + + #[test] + fn long_prefix_is_left_truncated_with_an_ellipsis() { + let matcher = compile_content_search_regex("needle", true, false, false).expect("regex"); + let line = format!("{}needle", "x".repeat(200)); + let (before, inside, _) = build_content_match_preview(&line, &matcher); + let before = before.expect("before segment"); + assert!(before.starts_with('\u{2026}')); + assert_eq!(before.chars().count(), MAX_PREVIEW_BEFORE_CHARS); + assert_eq!(inside.as_deref(), Some("needle")); + } + + #[test] + fn trailing_context_is_capped_by_the_total_budget() { + let matcher = compile_content_search_regex("needle", true, false, false).expect("regex"); + let line = format!("needle{}", "y".repeat(1000)); + let (before, inside, after) = build_content_match_preview(&line, &matcher); + let total = before.unwrap().chars().count() + + inside.unwrap().chars().count() + + after.unwrap().chars().count(); + assert_eq!(total, MAX_PREVIEW_CHARS); + } + + #[test] + fn multibyte_prefix_truncation_keeps_char_boundaries() { + let matcher = compile_content_search_regex("needle", true, false, false).expect("regex"); + let line = format!("{}needle", "中".repeat(100)); + let (before, inside, _) = build_content_match_preview(&line, &matcher); + let before = before.expect("before segment"); + assert_eq!(before.chars().count(), MAX_PREVIEW_BEFORE_CHARS); + assert!(before.chars().skip(1).all(|character| character == '中')); + assert_eq!(inside.as_deref(), Some("needle")); + } + + #[test] + fn zero_width_match_falls_back_to_the_matched_text() { + let matcher = compile_content_search_regex("x*", true, true, false).expect("regex"); + let (_, inside, _) = build_content_match_preview("abc", &matcher); + assert_eq!(inside.as_deref(), Some("")); + } +} diff --git a/src/crates/services/services-core/src/filesystem/mod.rs b/src/crates/services/services-core/src/filesystem/mod.rs index 155163a70e..16b8f4f074 100644 --- a/src/crates/services/services-core/src/filesystem/mod.rs +++ b/src/crates/services/services-core/src/filesystem/mod.rs @@ -5,6 +5,7 @@ //! `bitfun-core` may still layer remote-workspace routing or legacy error //! mapping on top of these primitives. +mod content_preview; mod error; mod factory; mod listing; @@ -13,6 +14,9 @@ mod service; mod tree; mod types; +pub use content_preview::{ + build_content_match_preview, compile_content_search_regex, ContentMatchPreviewBuilder, +}; pub use error::{FileSystemError, FileSystemResult}; pub use factory::FileSystemServiceFactory; pub use listing::{ diff --git a/src/crates/services/services-core/src/filesystem/tree.rs b/src/crates/services/services-core/src/filesystem/tree.rs index fd01ebedee..808e6d1ccf 100644 --- a/src/crates/services/services-core/src/filesystem/tree.rs +++ b/src/crates/services/services-core/src/filesystem/tree.rs @@ -2,11 +2,12 @@ //! //! Provides file tree building, directory scanning, and file search +use super::content_preview::{build_content_match_preview, compile_content_search_regex}; use super::error::{FileSystemError, FileSystemResult}; use log::warn; use ignore::WalkBuilder; -use regex::{Regex, RegexBuilder}; +use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs::File; @@ -1311,82 +1312,10 @@ impl FileTreeService { use_regex: bool, whole_word: bool, ) -> FileSystemResult { - let search_pattern = if use_regex { - pattern.to_string() - } else if whole_word { - format!(r"\b{}\b", regex::escape(pattern)) - } else { - regex::escape(pattern) - }; - - RegexBuilder::new(&search_pattern) - .case_insensitive(!case_sensitive) - .build() + compile_content_search_regex(pattern, case_sensitive, use_regex, whole_word) .map_err(|error| FileSystemError::service(format!("Invalid regex pattern: {}", error))) } - fn take_first_chars(text: &str, max_chars: usize) -> String { - if max_chars == 0 { - return String::new(); - } - - let mut end_index = text.len(); - for (char_count, (byte_index, _)) in text.char_indices().enumerate() { - if char_count == max_chars { - end_index = byte_index; - break; - } - } - - text[..end_index].to_string() - } - - fn left_truncate_with_ellipsis(text: &str, max_chars: usize) -> String { - let total_chars = text.chars().count(); - if total_chars <= max_chars { - return text.to_string(); - } - - if max_chars <= 1 { - return "\u{2026}".to_string(); - } - - let keep_chars = max_chars - 1; - let start_index = text - .char_indices() - .nth(total_chars.saturating_sub(keep_chars)) - .map(|(index, _)| index) - .unwrap_or(0); - - format!("\u{2026}{}", &text[start_index..]) - } - - fn build_content_match_preview( - line: &str, - matcher: &Regex, - ) -> (Option, Option, Option) { - const MAX_PREVIEW_CHARS: usize = 250; - const MAX_PREVIEW_BEFORE_CHARS: usize = 26; - - let Some(found_match) = matcher.find(line) else { - return (None, None, None); - }; - - let full_before = &line[..found_match.start()]; - let before = Self::left_truncate_with_ellipsis(full_before, MAX_PREVIEW_BEFORE_CHARS); - - let mut chars_remaining = MAX_PREVIEW_CHARS.saturating_sub(before.chars().count()); - let mut inside = Self::take_first_chars(found_match.as_str(), chars_remaining); - chars_remaining = chars_remaining.saturating_sub(inside.chars().count()); - let after = Self::take_first_chars(&line[found_match.end()..], chars_remaining); - - if inside.is_empty() { - inside = found_match.as_str().to_string(); - } - - (Some(before), Some(inside), Some(after)) - } - fn build_search_result_group(results: Vec) -> Option { let first = results.first()?.clone(); let file_name_match = results @@ -1501,7 +1430,7 @@ impl FileTreeService { } let (preview_before, preview_inside, preview_after) = - Self::build_content_match_preview(line, matcher); + build_content_match_preview(line, matcher); let line = line.to_string(); matched_results.push(FileSearchResult { diff --git a/src/crates/services/services-integrations/AGENTS.md b/src/crates/services/services-integrations/AGENTS.md index 666ae721f2..e28d8809a6 100644 --- a/src/crates/services/services-integrations/AGENTS.md +++ b/src/crates/services/services-integrations/AGENTS.md @@ -49,7 +49,11 @@ slices that are outside pure product logic but still platform-neutral. wrapper: `src/apps/desktop/src/api/relay_deploy_api.rs`. - Workspace search owns the local flashgrep daemon/session lifecycle and indexed-search result conversion behind `workspace-search`; product config - and workspace bootstrap stay in the core facade as injected hooks. + and workspace bootstrap stay in the core facade as injected hooks. The daemon + returns match positions only, so content output uses + `search/grouped_line_matches` and hydrates line text from disk in + `workspace_search/line_hydration.rs`; the preview primitives it shares with + the ripgrep path live in `bitfun-services-core::filesystem::content_preview`. - Remote SSH workspace-search owns the disabled surface, path/scope/probe, bundle/retry strategy, and flashgrep session/context lifecycle behind a provider boundary. diff --git a/src/crates/services/services-integrations/src/remote_ssh/workspace_search/mod.rs b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/mod.rs index 82116c018f..d2600a4868 100644 --- a/src/crates/services/services-integrations/src/remote_ssh/workspace_search/mod.rs +++ b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/mod.rs @@ -15,6 +15,7 @@ use std::path::{Path, PathBuf}; #[cfg(not(feature = "remote-ssh-concrete"))] pub mod disabled; +mod remote_line_hydration; mod service; pub use service::{ @@ -450,7 +451,6 @@ mod tests { with_line_matches.line_matches.push(LineMatch { path: "/repo/src/lib.rs".to_string(), line_number: 42, - line_text: Some("needle".to_string()), }); assert!(!should_retry_remote_scan_fallback_as_files_with_matches( SearchBackend::ScanFallback, diff --git a/src/crates/services/services-integrations/src/remote_ssh/workspace_search/remote_line_hydration.rs b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/remote_line_hydration.rs new file mode 100644 index 0000000000..c1425028e0 --- /dev/null +++ b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/remote_line_hydration.rs @@ -0,0 +1,588 @@ +//! Batched remote hydration of daemon line matches. +//! +//! The flashgrep daemon reports match *positions* only, so content output has to +//! read the matched lines from wherever the files live. Locally that is one +//! `File::open` per file (see `workspace_search::line_hydration`); over SSH the +//! same shape would be one round trip per file, which at a 250-match head limit +//! is 250 sequential round trips — unusable on any real link. +//! +//! So the remote path ships a *manifest* instead: the files and the exact line +//! numbers wanted in each, read by one `awk` pass on the far side. That is a +//! constant handful of round trips regardless of how many files matched, and the +//! text comes back already clamped so a pathological line cannot flood the link. +//! +//! Results are built through the same `push_line_results` the local path uses, +//! so an indexed match renders identically no matter which side read it. + +use crate::workspace_search::line_hydration::{ + plan_wanted_lines, push_line_results, MAX_HYDRATED_LINE_COLUMNS, +}; +use bitfun_services_core::filesystem::{ContentMatchPreviewBuilder, FileSearchResult}; + +/// Ceiling on lines hydrated in one remote content search, applied on top of any +/// caller limit. +/// +/// An unbounded `max_results` is a local-only luxury: there the cost of an extra +/// match is a disk read, here it is bytes on a link the user is waiting on. Lines +/// past this are dropped and reported as truncation, never silently. +pub(crate) const MAX_REMOTE_HYDRATED_LINES: usize = 1_000; + +/// Byte ceiling applied to each line on the remote side, before it is sent. +/// +/// Sized so it can never change what the user sees: the shared renderer clamps to +/// [`MAX_HYDRATED_LINE_COLUMNS`] *characters*, which is at most 4× that in UTF-8 +/// bytes, so anything this cuts was already past the display cut. +const MAX_REMOTE_LINE_BYTES: usize = MAX_HYDRATED_LINE_COLUMNS * 4 + 96; + +// Enforced at compile time rather than in a test: if the byte clamp could ever +// cut inside the first MAX_HYDRATED_LINE_COLUMNS characters, remote and local +// renderings would diverge on the part the user actually reads. +const _: () = assert!(MAX_REMOTE_LINE_BYTES > MAX_HYDRATED_LINE_COLUMNS * 4); + +/// Manifest budget for a single command. +/// +/// An SSH `exec` request carries its command in one protocol packet, and +/// implementations cap that in the tens of kilobytes; a repo-wide match list can +/// exceed it. Splitting keeps every command comfortably inside the limit while +/// staying at a handful of round trips in the worst case, and exactly one in the +/// common one. +const MAX_MANIFEST_COMMAND_BYTES: usize = 8 * 1024; + +const MANIFEST_HEREDOC_DELIMITER: &str = "__BITFUN_LINE_MANIFEST_EOF__"; + +/// Reads the matched lines for one file, positionally aligned with its wanted +/// line numbers. +#[derive(Debug, Default, Clone)] +pub(crate) struct RemoteFileLines { + pub texts: Vec>, + /// The file could not be opened at all — removed or made unreadable since the + /// snapshot the daemon answered from. Distinct from a line past end of file, + /// which is a readable file that simply got shorter. + pub unreadable: bool, +} + +#[derive(Debug, Default)] +pub(crate) struct RemoteHydratedLines { + pub results: Vec, + pub dropped_lines: usize, + pub unreadable_files: usize, +} + +/// The files and line numbers a remote content search will actually read, after +/// both the caller's limit and [`MAX_REMOTE_HYDRATED_LINES`] have been applied. +pub(crate) fn plan_remote_hydration( + files: &[(String, Vec)], + max_results: Option, +) -> (Vec<(String, Vec)>, usize) { + let budget = max_results + .unwrap_or(MAX_REMOTE_HYDRATED_LINES) + .min(MAX_REMOTE_HYDRATED_LINES); + plan_wanted_lines(files, Some(budget)) +} + +/// Splits the planned files into manifest chunks, each small enough to travel as +/// one command. Returns file index ranges, not paths, so callers keep the +/// daemon's ordering. +pub(crate) fn chunk_manifest(files: &[(String, Vec)]) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut current: Vec = Vec::new(); + let mut current_bytes = 0usize; + + for (index, (path, wanted)) in files.iter().enumerate() { + if wanted.is_empty() || path.contains('\n') || path.contains('\t') { + // Tab and newline are the manifest's own framing. A path containing + // either cannot be expressed, and is left out of every chunk so it + // surfaces as unreadable rather than corrupting its neighbours. + continue; + } + let entry_bytes = manifest_line(index, path, wanted).len(); + if !current.is_empty() && current_bytes + entry_bytes > MAX_MANIFEST_COMMAND_BYTES { + chunks.push(std::mem::take(&mut current)); + current_bytes = 0; + } + current.push(index); + current_bytes += entry_bytes; + } + + if !current.is_empty() { + chunks.push(current); + } + chunks +} + +fn manifest_line(index: usize, path: &str, wanted: &[usize]) -> String { + let lines = wanted + .iter() + .map(|line| line.to_string()) + .collect::>() + .join(","); + format!("{index}\t{lines}\t{path}\n") +} + +/// Builds the one command that reads every line the chunk asks for. +/// +/// `awk` rather than a shell loop because the read has to be a single pass per +/// file and stop at the last wanted line — a `sed`/`head` composition would +/// re-open, and a shell `while read` would be a fork per line. `LC_ALL=C` keeps +/// `length`/`substr` byte-oriented so the clamp is predictable on non-UTF-8 input. +pub(crate) fn build_read_command(files: &[(String, Vec)], chunk: &[usize]) -> String { + let mut manifest = String::new(); + for index in chunk { + let (path, wanted) = &files[*index]; + manifest.push_str(&manifest_line(*index, path, wanted)); + } + + format!( + "LC_ALL=C awk -v MAXB={max_bytes} '{program}' <<'{delimiter}'\n{manifest}{delimiter}\n", + max_bytes = MAX_REMOTE_LINE_BYTES, + program = AWK_READ_PROGRAM, + delimiter = MANIFEST_HEREDOC_DELIMITER, + ) +} + +/// Reads `idxlinespath` records and emits `idxLlinetext` +/// per hydrated line, or a single `idxE0` when the file could not +/// be opened. The path is last in the record so it is the only field that may +/// contain a tab, and the text is last in the output for the same reason. +/// +/// The inner `want[k] < cur` skip is a guard, not a hot path: for the +/// sorted-unique 1-based lines `plan_wanted_lines` produces it never advances. +/// It is what keeps a line number the walker could never reach — a 0, say — from +/// stalling the cursor against a `want` entry that will never equal `cur`. +const AWK_READ_PROGRAM: &str = concat!( + "{", + "t1 = index($0, \"\\t\"); if (t1 < 2) next;", + "rest = substr($0, t1 + 1);", + "t2 = index(rest, \"\\t\"); if (t2 < 2) next;", + "idx = substr($0, 1, t1 - 1);", + "n = split(substr(rest, 1, t2 - 1), want, \",\"); if (n < 1) next;", + "path = substr(rest, t2 + 1);", + "cur = 0; k = 1; rc = 1;", + "while (k <= n) {", + "rc = (getline line < path); if (rc <= 0) break;", + "cur++;", + "while (k <= n && want[k] + 0 < cur) k++;", + "if (k <= n && want[k] + 0 == cur) {", + "if (length(line) > MAXB) line = substr(line, 1, MAXB);", + "print idx \"\\tL\\t\" cur \"\\t\" line; k++;", + "}", + "}", + "close(path);", + "if (rc < 0) print idx \"\\tE\\t0\\t\";", + "}", +); + +/// Folds one command's stdout back onto the planned files. +/// +/// `into` is indexed by the planned file index, so a chunk only ever fills its +/// own slots and a failed chunk leaves the rest untouched. +pub(crate) fn absorb_read_output( + stdout: &str, + files: &[(String, Vec)], + into: &mut [RemoteFileLines], +) { + for record in stdout.split('\n') { + if record.is_empty() { + continue; + } + let Some((index, kind, line_number, text)) = split_record(record) else { + continue; + }; + let Some(slot) = into.get_mut(index) else { + continue; + }; + if kind == "E" { + slot.unreadable = true; + continue; + } + let Some((_, wanted)) = files.get(index) else { + continue; + }; + let Ok(position) = wanted.binary_search(&line_number) else { + // A line the manifest never asked for: ignore rather than trust it. + continue; + }; + if slot.texts.len() != wanted.len() { + slot.texts = vec![None; wanted.len()]; + } + slot.texts[position] = Some(text.trim_end_matches('\r').to_string()); + } +} + +fn split_record(record: &str) -> Option<(usize, &str, usize, &str)> { + let (index, rest) = record.split_once('\t')?; + let (kind, rest) = rest.split_once('\t')?; + let (line_number, text) = rest.split_once('\t')?; + Some(( + index.parse().ok()?, + kind, + line_number.parse().unwrap_or(0), + text, + )) +} + +/// Turns the per-file reads into content results, in the daemon's file order. +pub(crate) fn build_remote_results( + files: &[(String, Vec)], + reads: Vec, + dropped_lines: usize, + preview: Option<&ContentMatchPreviewBuilder>, +) -> RemoteHydratedLines { + let mut outcome = RemoteHydratedLines { + dropped_lines, + ..RemoteHydratedLines::default() + }; + + for ((path, wanted), read) in files.iter().zip(reads) { + if read.unreadable { + outcome.unreadable_files += 1; + } + let mut texts = read.texts; + texts.resize(wanted.len(), None); + push_line_results(path, wanted, texts, preview, &mut outcome.results); + } + + outcome +} + +#[cfg(test)] +mod tests { + use super::*; + + fn files(entries: &[(&str, &[usize])]) -> Vec<(String, Vec)> { + entries + .iter() + .map(|(path, lines)| ((*path).to_string(), lines.to_vec())) + .collect() + } + + #[test] + fn the_remote_ceiling_applies_even_without_a_caller_limit() { + let wanted = (1..=MAX_REMOTE_HYDRATED_LINES + 25).collect::>(); + let (planned, dropped) = plan_remote_hydration(&files(&[("/repo/a.rs", &wanted)]), None); + + assert_eq!(planned[0].1.len(), MAX_REMOTE_HYDRATED_LINES); + assert_eq!(dropped, 25); + } + + #[test] + fn a_caller_limit_below_the_ceiling_still_wins() { + let (planned, dropped) = + plan_remote_hydration(&files(&[("/repo/a.rs", &[1, 2, 3, 4])]), Some(2)); + + assert_eq!(planned[0].1, vec![1, 2]); + assert_eq!(dropped, 2); + } + + #[test] + fn one_command_covers_a_head_limit_sized_result() { + // 250 matches spread one per file is the shape a default `Grep` produces. + let paths = (0..250) + .map(|index| format!("/workspace/project/src/module/file_{index}.rs")) + .collect::>(); + let planned = paths + .iter() + .map(|path| (path.clone(), vec![42])) + .collect::>(); + + let chunks = chunk_manifest(&planned); + + assert!( + chunks.len() <= 4, + "a 250-match search must stay at a handful of round trips, got {}", + chunks.len() + ); + let covered = chunks.iter().map(Vec::len).sum::(); + assert_eq!(covered, 250); + for chunk in &chunks { + assert!(build_read_command(&planned, chunk).len() < 32 * 1024); + } + } + + #[test] + fn chunks_partition_the_files_in_order() { + let planned = (0..400) + .map(|index| { + ( + format!("/very/long/remote/path/segment/file_{index}.rs"), + vec![1, 2, 3], + ) + }) + .collect::>(); + + let chunks = chunk_manifest(&planned); + + assert!(chunks.len() > 1, "the fixture is meant to need splitting"); + let flattened = chunks.concat(); + assert_eq!(flattened, (0..400).collect::>()); + } + + #[test] + fn a_path_that_cannot_be_framed_is_left_out_of_every_chunk() { + let planned = files(&[ + ("/repo/ok.rs", &[1]), + ("/repo/we\tird.rs", &[1]), + ("/repo/also\nbad.rs", &[1]), + ("/repo/fine.rs", &[2]), + ]); + + assert_eq!(chunk_manifest(&planned).concat(), vec![0, 3]); + } + + #[test] + fn the_command_carries_every_wanted_line_of_its_chunk() { + let planned = files(&[("/repo/a.rs", &[3, 9]), ("/repo/b.rs", &[1])]); + + let command = build_read_command(&planned, &[0, 1]); + + assert!(command.contains("0\t3,9\t/repo/a.rs\n")); + assert!(command.contains("1\t1\t/repo/b.rs\n")); + assert!(command.ends_with(&format!("{MANIFEST_HEREDOC_DELIMITER}\n"))); + // Quoted heredoc: the manifest must not be expanded by the remote shell. + assert!(command.contains(&format!("<<'{MANIFEST_HEREDOC_DELIMITER}'"))); + } + + #[test] + fn output_lands_in_the_slot_its_line_number_asked_for() { + let planned = files(&[("/repo/a.rs", &[3, 9]), ("/repo/b.rs", &[1])]); + let mut reads = vec![RemoteFileLines::default(); planned.len()]; + + absorb_read_output( + "0\tL\t9\tnine\n0\tL\t3\tthree\n1\tE\t0\t\n", + &planned, + &mut reads, + ); + + assert_eq!( + reads[0].texts, + vec![Some("three".to_string()), Some("nine".to_string())] + ); + assert!(!reads[0].unreadable); + assert!(reads[1].unreadable); + } + + #[test] + fn text_containing_tabs_survives_the_framing() { + let planned = files(&[("/repo/a.rs", &[1])]); + let mut reads = vec![RemoteFileLines::default(); 1]; + + absorb_read_output("0\tL\t1\tlet\tx\t= 1;\r\n", &planned, &mut reads); + + assert_eq!(reads[0].texts[0].as_deref(), Some("let\tx\t= 1;")); + } + + #[test] + fn a_line_the_manifest_never_asked_for_is_ignored() { + let planned = files(&[("/repo/a.rs", &[1])]); + let mut reads = vec![RemoteFileLines::default(); 1]; + + absorb_read_output( + "0\tL\t7\tsurprise\n99\tL\t1\tout of range\n", + &planned, + &mut reads, + ); + + // Nothing was absorbed, so the file renders as a bare position. + assert!(reads[0].texts.iter().all(Option::is_none)); + let outcome = build_remote_results(&planned, reads, 0, None); + assert_eq!(outcome.results.len(), 1); + assert_eq!(outcome.results[0].matched_content, None); + } + + #[test] + fn a_chunk_that_never_answered_keeps_its_positions() { + let planned = files(&[("/repo/a.rs", &[4, 8])]); + let reads = vec![RemoteFileLines::default()]; + + let outcome = build_remote_results(&planned, reads, 0, None); + + let rendered = outcome + .results + .iter() + .map(|result| (result.line_number, result.matched_content.clone())) + .collect::>(); + assert_eq!(rendered, vec![(Some(4), None), (Some(8), None)]); + assert_eq!(outcome.unreadable_files, 0); + } + + #[test] + fn long_lines_are_clamped_the_same_way_the_local_path_clamps_them() { + let planned = files(&[("/repo/a.rs", &[1])]); + let long = "x".repeat(MAX_HYDRATED_LINE_COLUMNS * 3); + let reads = vec![RemoteFileLines { + texts: vec![Some(long)], + unreadable: false, + }]; + + let outcome = build_remote_results(&planned, reads, 0, None); + + let content = outcome.results[0] + .matched_content + .as_deref() + .expect("content"); + assert!(content.ends_with(" [truncated]")); + assert!(content.starts_with(&"x".repeat(MAX_HYDRATED_LINE_COLUMNS))); + } +} + +/// Executes the command this module generates against a real POSIX shell. +/// +/// The Rust half of the protocol is covered above; this is the other half. An +/// `awk` slip or a heredoc framing mistake only ever shows up when the thing +/// actually runs, and on the remote path the only place it would show up is a +/// user's SSH session. +#[cfg(all(test, unix))] +mod shell_tests { + use super::*; + use std::fs; + use std::process::Command; + + fn run(files: &[(String, Vec)]) -> Vec { + let mut reads = vec![RemoteFileLines::default(); files.len()]; + for chunk in chunk_manifest(files) { + let output = Command::new("sh") + .arg("-c") + .arg(build_read_command(files, &chunk)) + .output() + .expect("run the generated line-read command"); + assert!( + output.status.success(), + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + absorb_read_output(&String::from_utf8_lossy(&output.stdout), files, &mut reads); + } + reads + } + + #[test] + fn the_generated_command_reads_exactly_the_wanted_lines() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = |name: &str| dir.path().join(name).to_string_lossy().to_string(); + + fs::write( + dir.path().join("a.txt"), + "alpha +needle one +gamma +needle two +", + ) + .expect("write"); + fs::write( + dir.path().join("crlf.txt"), + "第一行 needle +第二行 +", + ) + .expect("write"); + fs::write( + dir.path().join("tabs.txt"), + "let x = 1; +only line +", + ) + .expect("write"); + fs::write( + dir.path().join("short.txt"), + "short +", + ) + .expect("write"); + fs::write(dir.path().join("empty.txt"), "").expect("write"); + fs::write( + dir.path().join("long.txt"), + format!("needle{}\n", "x".repeat(MAX_REMOTE_LINE_BYTES * 2)), + ) + .expect("write"); + + let files = vec![ + (path("a.txt"), vec![2, 4]), + (path("crlf.txt"), vec![1]), + (path("tabs.txt"), vec![1, 2]), + // Line 9 is past end of file: the snapshot can be ahead of the worktree. + (path("short.txt"), vec![1, 9]), + (path("empty.txt"), vec![1]), + (path("long.txt"), vec![1]), + (path("gone.txt"), vec![1]), + ]; + + let reads = run(&files); + + assert_eq!( + reads[0].texts, + vec![ + Some("needle one".to_string()), + Some("needle two".to_string()) + ] + ); + // The carriage return is stripped on the way in, exactly as the local + // reader strips it, and multi-byte text survives the byte-oriented awk. + assert_eq!(reads[1].texts, vec![Some("第一行 needle".to_string())]); + assert_eq!( + reads[2].texts, + vec![ + Some("let x = 1;".to_string()), + Some("only line".to_string()) + ] + ); + assert_eq!(reads[3].texts, vec![Some("short".to_string()), None]); + assert!( + !reads[3].unreadable, + "a short file is readable, just shorter" + ); + assert!(!reads[4].unreadable, "an empty file is readable"); + assert!(reads[4].texts.iter().all(Option::is_none)); + assert_eq!( + reads[5].texts[0].as_ref().map(String::len), + Some(MAX_REMOTE_LINE_BYTES), + "the remote clamp bounds what crosses the link" + ); + assert!( + reads[6].unreadable, + "a missing file must be reported as such" + ); + } + + #[test] + fn the_generated_command_renders_the_same_text_the_local_reader_would() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("sample.rs"); + fs::write( + &file, + "fn main() { + let needle = 1; +} +", + ) + .expect("write"); + let path = file.to_string_lossy().to_string(); + + let files = vec![(path.clone(), vec![2])]; + let preview = + ContentMatchPreviewBuilder::new("needle", true, false, false).expect("preview builder"); + + let remote = build_remote_results(&files, run(&files), 0, Some(&preview)); + let local = crate::workspace_search::line_hydration::hydrate_grouped_line_matches( + &files, + None, + Some(&preview), + ); + + let render = |result: &FileSearchResult| { + ( + result.path.clone(), + result.name.clone(), + result.line_number, + result.matched_content.clone(), + result.preview_before.clone(), + result.preview_inside.clone(), + result.preview_after.clone(), + ) + }; + assert_eq!( + remote.results.iter().map(render).collect::>(), + local.results.iter().map(render).collect::>() + ); + } +} diff --git a/src/crates/services/services-integrations/src/remote_ssh/workspace_search/service.rs b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/service.rs index ba51415fdf..7d9324f9fa 100644 --- a/src/crates/services/services-integrations/src/remote_ssh/workspace_search/service.rs +++ b/src/crates/services/services-integrations/src/remote_ssh/workspace_search/service.rs @@ -1,3 +1,7 @@ +use super::remote_line_hydration::{ + absorb_read_output, build_read_command, build_remote_results, chunk_manifest, + plan_remote_hydration, RemoteFileLines, +}; use super::{ build_remote_scope, join_remote_path, local_flashgrep_bundle_for_arch, looks_like_linux_workspace_root, parse_remote_architecture_output, parse_remote_os_output, @@ -9,18 +13,20 @@ use crate::remote_ssh::{normalize_remote_workspace_path, RemoteWorkspaceEntry}; use crate::workspace_search::flashgrep::error::AppError; use crate::workspace_search::flashgrep::{ drain_content_length_messages, log_flashgrep_stderr_line_with_context, ClientCapabilities, - ClientInfo, FlashgrepRepoSession, GlobOutcome, GlobParams, GlobRequest, InitializeParams, - OpenRepoParams, ProtocolClient, QuerySpec, RefreshPolicyConfig, RepoConfig, RepoRef, - RepoStatus, Request, Response, SearchBackend, SearchModeConfig, SearchOutcome, SearchParams, - SearchRequest, SearchResults, TaskRef, TaskStatus, FLASHGREP_LOG_TARGET, + ClientInfo, FlashgrepRepoSession, GlobOutcome, GlobParams, GlobRequest, + GroupedLineMatchResults, InitializeParams, OpenRepoParams, ProtocolClient, QuerySpec, + RefreshPolicyConfig, RepoConfig, RepoRef, RepoStatus, Request, Response, SearchBackend, + SearchModeConfig, SearchOutcome, SearchParams, SearchRequest, SearchResults, TaskRef, + TaskStatus, FLASHGREP_LOG_TARGET, }; use crate::workspace_search::result_mapping::convert_search_results; use crate::workspace_search::{ - ContentSearchRequest, ContentSearchResult, GlobSearchRequest, GlobSearchResult, - IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchFileCount, WorkspaceSearchRepoStatus, + ContentSearchOutputMode, ContentSearchRequest, ContentSearchResult, GlobSearchRequest, + GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchFileCount, + WorkspaceSearchRepoStatus, }; use async_trait::async_trait; -use bitfun_services_core::filesystem::FileSearchOutcome; +use bitfun_services_core::filesystem::{ContentMatchPreviewBuilder, FileSearchOutcome}; use std::collections::HashMap; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; @@ -135,6 +141,9 @@ struct RemoteStdioRepoSession { struct RemoteStdioDaemonClient { protocol: ProtocolClient, + /// Kept so a session can run plain remote commands — line hydration reads the + /// matched files over the same connection the daemon is speaking on. + connection_id: String, } struct RemoteStdioOperationLease { @@ -185,7 +194,10 @@ impl RemoteStdioDaemonClient { .spawn_stdio_daemon(&connection_id, &command, write_rx, stdio_protocol) .await?; - let client = Arc::new(Self { protocol }); + let client = Arc::new(Self { + protocol, + connection_id, + }); client.initialize().await?; Ok(client) } @@ -354,7 +366,6 @@ impl RemoteStdioRepoSession { repo_id: self.repo_id.clone(), query, scope, - allow_scan_fallback: true, }, }) .await? @@ -371,6 +382,40 @@ impl RemoteStdioRepoSession { } } + /// Runs the per-file grouped variant of search. + /// + /// Content output needs line text, which no daemon search mode returns, and + /// the grouping is what lets the remote reader ask for every file exactly + /// once instead of once per matched line. + async fn search_grouped_line_matches( + &self, + query: QuerySpec, + scope: crate::workspace_search::flashgrep::PathScope, + ) -> Result<(SearchBackend, RepoStatus, GroupedLineMatchResults), String> { + let _lease = self.acquire_operation(); + match self + .client + .send_request(Request::SearchGroupedLineMatches { + params: SearchParams { + repo_id: self.repo_id.clone(), + query, + scope, + }, + }) + .await? + { + Response::SearchGroupedLineMatchesCompleted { + backend, + status, + results, + .. + } => Ok((backend, status, results)), + other => Err(format!( + "Unexpected remote flashgrep search/grouped_line_matches response: {other:?}" + )), + } + } + async fn glob( &self, scope: crate::workspace_search::flashgrep::PathScope, @@ -475,6 +520,15 @@ pub struct RemoteWorkspaceSearchService { preferred_connection_id: Option, } +/// The parts of a content request the preview highlighter needs, captured before +/// the request is consumed building the daemon scope. +struct RemoteContentPreviewSpec { + pattern: String, + case_sensitive: bool, + use_regex: bool, + whole_word: bool, +} + #[derive(Debug, Clone)] struct RemoteSearchContext { connection: RemoteWorkspaceEntry, @@ -518,6 +572,9 @@ impl RemoteWorkspaceSearchService { Ok(WorkspaceIndexStatus { active_task, repo_status, + // Remote workspaces are indexed by the remote daemon directly; BitFun's local + // auto-index policy never evaluates them, so there is no decision to report. + auto_index: None, }) } @@ -556,6 +613,12 @@ impl RemoteWorkspaceSearchService { )?; let max_results = request.max_results.filter(|limit| *limit > 0); let primary_search_mode = remote_stdio_search_mode(request.output_mode); + let preview_spec = RemoteContentPreviewSpec { + pattern: request.pattern.clone(), + case_sensitive: request.case_sensitive, + use_regex: request.use_regex, + whole_word: request.whole_word, + }; let query = QuerySpec { pattern: request.pattern.clone(), patterns: Vec::new(), @@ -565,8 +628,6 @@ impl RemoteWorkspaceSearchService { fixed_strings: !request.use_regex, word_regexp: request.whole_word, line_regexp: false, - before_context: request.before_context, - after_context: request.after_context, top_k_tokens: 6, max_count: None, global_max_results: max_results, @@ -574,6 +635,38 @@ impl RemoteWorkspaceSearchService { }; let output_mode = request.output_mode; + + if matches!(output_mode, ContentSearchOutputMode::Content) { + match self + .search_content_hydrated( + &session, + preview_spec, + query.clone(), + scope.clone(), + max_results, + ) + .await + { + Ok(Some(result)) => return Ok(result), + Ok(None) => { + // The daemon answered without a per-file grouping — the scan + // fallback does that when it can only report totals. Fall + // through to the plain search below, which already knows how + // to recover a file list from that case. + log::info!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search falling back to positions-only search: repo_root={repo_root}" + ); + } + Err(error) => { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search grouped query failed, falling back to positions-only search: repo_root={repo_root}, error={error}" + ); + } + } + } + let (backend, repo_status, mut raw_results) = session.search(query, scope.clone()).await?; if should_retry_remote_scan_fallback_as_files_with_matches( backend, @@ -596,8 +689,6 @@ impl RemoteWorkspaceSearchService { fixed_strings: !request.use_regex, word_regexp: request.whole_word, line_regexp: false, - before_context: request.before_context, - after_context: request.after_context, top_k_tokens: 6, max_count: None, global_max_results: max_results, @@ -666,6 +757,140 @@ impl RemoteWorkspaceSearchService { }) } + /// Content output for a remote repo: grouped match positions from the daemon, + /// line text batched back over SSH. + /// + /// Returns `Ok(None)` when the daemon reported matches but no per-file + /// grouping — the scan fallback does that when it can only produce totals — + /// so the caller can fall back to the positions-only path that already knows + /// how to recover a file list from that answer. + async fn search_content_hydrated( + &self, + session: &RemoteStdioSessionLease, + preview_spec: RemoteContentPreviewSpec, + query: QuerySpec, + scope: crate::workspace_search::flashgrep::PathScope, + max_results: Option, + ) -> Result, String> { + let (backend, repo_status, grouped) = session + .search_grouped_line_matches(query, scope) + .await + .map_err(|error| format!("Remote content search failed: {error}"))?; + + if grouped.files.is_empty() && grouped.matched_lines > 0 { + return Ok(None); + } + + let (planned, dropped_lines) = plan_remote_hydration(&grouped.files, max_results); + let reads = self + .read_remote_lines(&session.client.connection_id, &planned) + .await; + + let hydrated = tokio::task::spawn_blocking(move || { + // A pattern the daemon accepts can still fail to compile here + // (different regex dialect, or a multiline query no single line + // matches), in which case results keep their line text and lose only + // the highlight. + let preview = match ContentMatchPreviewBuilder::new( + &preview_spec.pattern, + preview_spec.case_sensitive, + preview_spec.use_regex, + preview_spec.whole_word, + ) { + Ok(preview) => Some(preview), + Err(error) => { + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search preview highlighting disabled: {error}" + ); + None + } + }; + build_remote_results(&planned, reads, dropped_lines, preview.as_ref()) + }) + .await + .map_err(|error| format!("Remote content search line hydration failed: {error}"))?; + + if hydrated.unreadable_files > 0 { + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search could not read {} matched file(s); reporting their matches without line text", + hydrated.unreadable_files + ); + } + + Ok(Some(ContentSearchResult { + outcome: FileSearchOutcome { + results: hydrated.results, + truncated: grouped.limit_reached || hydrated.dropped_lines > 0, + }, + // `search/grouped_line_matches` reports per-file counts only for its + // top-10 summary, which is not the full list the count output mode + // promises, so it is left to that mode. + file_counts: Vec::new(), + hits: Vec::new(), + backend: backend.into(), + repo_status: repo_status.into(), + candidate_docs: grouped.candidate_docs, + matched_lines: grouped.matched_lines, + matched_occurrences: grouped.matched_occurrences, + })) + } + + /// Reads every wanted line of every planned file in a handful of commands. + /// + /// Failure is per-chunk and never fatal: a chunk that errors leaves its files + /// without text, which renders as a bare `path:line` locator — the same thing + /// the whole remote path did before hydration existed. Losing highlighting on + /// part of a result set beats failing a search that already found its matches. + async fn read_remote_lines( + &self, + connection_id: &str, + planned: &[(String, Vec)], + ) -> Vec { + let mut reads = vec![RemoteFileLines::default(); planned.len()]; + let chunks = chunk_manifest(planned); + if chunks.is_empty() { + return reads; + } + + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search hydrating {} file(s) in {} command(s)", + planned.len(), + chunks.len() + ); + + for chunk in chunks { + let command = build_read_command(planned, &chunk); + match self.provider.execute_command(connection_id, &command).await { + Ok(output) => { + if output.exit_code != 0 { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search line read exited {}: files={}, stderr={}", + output.exit_code, + chunk.len(), + output.stderr.trim() + ); + } + // Absorbed either way: awk writes as it goes, so a command + // that died partway through still produced usable records. + absorb_read_output(&output.stdout, planned, &mut reads); + } + Err(error) => { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Remote workspace content search line read failed: files={}, error={error}", + chunk.len() + ); + } + } + } + + reads + } + pub async fn glob(&self, request: GlobSearchRequest) -> Result { let repo_root = normalize_remote_workspace_path(&request.repo_root.to_string_lossy()); let session = self.get_or_open_stdio_session(&repo_root).await?; diff --git a/src/crates/services/services-integrations/src/workspace_search/auto_index.rs b/src/crates/services/services-integrations/src/workspace_search/auto_index.rs new file mode 100644 index 0000000000..3444438f99 --- /dev/null +++ b/src/crates/services/services-integrations/src/workspace_search/auto_index.rs @@ -0,0 +1,240 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +use tokio::task::spawn_blocking; + +pub(crate) const DEFAULT_AUTO_INDEX_MIN_FILES: usize = 2_000; +#[derive(Debug, Clone, Copy)] +pub(crate) struct AutoIndexPolicy { + pub min_indexable_files: usize, + pub max_file_size: u64, +} + +impl Default for AutoIndexPolicy { + fn default() -> Self { + Self { + min_indexable_files: DEFAULT_AUTO_INDEX_MIN_FILES, + max_file_size: 50 * 1024 * 1024, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum AutoIndexDecision { + /// Counting stops as soon as the threshold is reached, so this is a lower + /// bound rather than the workspace's real indexable file count. + Eligible { + indexable_files_at_least: usize, + }, + BelowThreshold { + indexable_files: usize, + }, + Unsupported { + reason: String, + }, +} + +pub(crate) async fn evaluate(repo_root: PathBuf, policy: AutoIndexPolicy) -> AutoIndexDecision { + match spawn_blocking(move || evaluate_blocking(&repo_root, policy)).await { + Ok(decision) => decision, + Err(error) => AutoIndexDecision::Unsupported { + reason: format!("file-count worker failed: {error}"), + }, + } +} + +fn evaluate_blocking(repo_root: &Path, policy: AutoIndexPolicy) -> AutoIndexDecision { + if policy.min_indexable_files == 0 { + return AutoIndexDecision::Eligible { + indexable_files_at_least: 0, + }; + } + + match git_indexable_file_count(repo_root, policy) { + Ok(count) => decision_for_count(count, policy.min_indexable_files), + Err(reason) => AutoIndexDecision::Unsupported { reason }, + } +} + +fn decision_for_count(count: usize, threshold: usize) -> AutoIndexDecision { + if count >= threshold { + AutoIndexDecision::Eligible { + indexable_files_at_least: count, + } + } else { + AutoIndexDecision::BelowThreshold { + indexable_files: count, + } + } +} + +/// Counts indexable files in two passes so that large workspaces never pay for +/// the untracked-file walk. +/// +/// `--others` re-scans the whole worktree and dominates the cost: on +/// chromium-src it takes ~4.2 s against ~89 ms for `--cached` alone, which is +/// the same worktree scan the daemon already pays inside `open_repo`. Any +/// workspace that reaches the threshold on tracked files alone therefore skips +/// the second pass entirely; only genuinely small (or freshly cloned, mostly +/// untracked) workspaces need it, and there the walk is cheap. +fn git_indexable_file_count(repo_root: &Path, policy: AutoIndexPolicy) -> Result { + let worktree_root = git_worktree_root(repo_root)?; + let tracked = git_ls_files_indexable_count(&worktree_root, &["--cached"], policy, 0)?; + if tracked >= policy.min_indexable_files { + return Ok(tracked); + } + git_ls_files_indexable_count( + &worktree_root, + &["--others", "--exclude-standard"], + policy, + tracked, + ) +} + +/// Runs one `git ls-files` selector and adds its indexable files to `carried`, +/// stopping as soon as the threshold is reached. +fn git_ls_files_indexable_count( + worktree_root: &Path, + selectors: &[&str], + policy: AutoIndexPolicy, + carried: usize, +) -> Result { + let output = Command::new("git") + .arg("ls-files") + .args(selectors) + .arg("-z") + .current_dir(worktree_root) + .output() + .map_err(|error| format!("git ls-files unavailable: {error}"))?; + if !output.status.success() { + return Err(format!("git ls-files exited with {}", output.status)); + } + + let mut count = carried; + for path in output.stdout.split(|byte| *byte == 0) { + if path.is_empty() { + continue; + } + let relative_path = String::from_utf8_lossy(path); + let absolute_path = worktree_root.join(relative_path.as_ref()); + if is_indexable_file(&absolute_path, policy.max_file_size) { + count += 1; + if count >= policy.min_indexable_files { + break; + } + } + } + Ok(count) +} + +fn git_worktree_root(repo_root: &Path) -> Result { + let output = Command::new("git") + .args(["rev-parse", "--show-toplevel"]) + .current_dir(repo_root) + .output() + .map_err(|error| format!("git worktree discovery unavailable: {error}"))?; + if !output.status.success() { + return Err(format!( + "git worktree discovery exited with {}", + output.status + )); + } + let root = String::from_utf8_lossy(&output.stdout); + let root = root.trim(); + if root.is_empty() { + return Err("git worktree discovery returned an empty root".to_string()); + } + let worktree_root = dunce::canonicalize(root) + .map_err(|error| format!("cannot canonicalize Git worktree root: {error}"))?; + let head = Command::new("git") + .args(["rev-parse", "--verify", "HEAD^{commit}"]) + .current_dir(&worktree_root) + .output() + .map_err(|error| format!("Git HEAD discovery unavailable: {error}"))?; + if !head.status.success() { + return Err("flashgrep requires a Git worktree with a HEAD commit".to_string()); + } + Ok(worktree_root) +} + +fn is_indexable_file(path: &Path, max_file_size: u64) -> bool { + std::fs::metadata(path) + .map(|metadata| metadata.is_file() && metadata.len() <= max_file_size) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::write; + use tempfile::TempDir; + + fn policy(min_indexable_files: usize, max_file_size: u64) -> AutoIndexPolicy { + AutoIndexPolicy { + min_indexable_files, + max_file_size, + } + } + + #[test] + fn git_count_uses_visible_tracked_and_untracked_files_and_size_limit() { + let repo = TempDir::new().expect("temp repo"); + Command::new("git") + .args(["init", "--quiet"]) + .current_dir(repo.path()) + .status() + .expect("git should be available"); + write(repo.path().join("tracked.txt"), "tracked").expect("write tracked"); + write(repo.path().join("untracked.txt"), "untracked").expect("write untracked"); + write(repo.path().join("large.bin"), vec![0_u8; 160]).expect("write large file"); + Command::new("git") + .args(["add", "tracked.txt"]) + .current_dir(repo.path()) + .status() + .expect("git add should work"); + Command::new("git") + .args([ + "-c", + "user.name=BitFun Test", + "-c", + "user.email=bitfun-test@example.invalid", + "commit", + "--quiet", + "-m", + "initial", + ]) + .current_dir(repo.path()) + .status() + .expect("git commit should work"); + + assert_eq!( + git_indexable_file_count(repo.path(), policy(10, 100)), + Ok(2) + ); + assert_eq!(git_indexable_file_count(repo.path(), policy(10, 8)), Ok(1)); + + // Tracked files alone reach this threshold, so the untracked walk is + // skipped and `untracked.txt` is never counted. + assert_eq!(git_indexable_file_count(repo.path(), policy(1, 100)), Ok(1)); + } + + #[test] + fn non_git_workspaces_are_explicitly_unsupported() { + let repo = TempDir::new().expect("temp repo"); + let decision = evaluate_blocking(repo.path(), policy(2, 100)); + assert!(matches!(decision, AutoIndexDecision::Unsupported { .. })); + } + + #[test] + fn git_workspaces_without_a_head_are_unsupported() { + let repo = TempDir::new().expect("temp repo"); + Command::new("git") + .args(["init", "--quiet"]) + .current_dir(repo.path()) + .status() + .expect("git should be available"); + + let decision = evaluate_blocking(repo.path(), policy(1, 100)); + assert!(matches!(decision, AutoIndexDecision::Unsupported { .. })); + } +} diff --git a/src/crates/services/services-integrations/src/workspace_search/flashgrep/client.rs b/src/crates/services/services-integrations/src/workspace_search/flashgrep/client.rs index 44c81c2c1b..7ae3328004 100644 --- a/src/crates/services/services-integrations/src/workspace_search/flashgrep/client.rs +++ b/src/crates/services/services-integrations/src/workspace_search/flashgrep/client.rs @@ -27,8 +27,8 @@ use super::{ repo_session::FlashgrepRepoSession, rpc_client::{read_content_length_message, ProtocolClient}, types::{ - GlobOutcome, GlobRequest, OpenRepoParams, RepoStatus, SearchOutcome, SearchRequest, - TaskStatus, + GlobOutcome, GlobRequest, GroupedLineMatchOutcome, OpenRepoParams, RepoStatus, + SearchOutcome, SearchRequest, TaskStatus, }, FLASHGREP_LOG_TARGET, }; @@ -273,7 +273,6 @@ impl RepoSession { repo_id: self.repo_id.clone(), query: request.query, scope: request.scope, - allow_scan_fallback: request.allow_scan_fallback, }, }, |response| match response { @@ -294,6 +293,44 @@ impl RepoSession { .await } + /// Runs a line-match search that returns matches grouped per file. + /// + /// Preferred over `search` for content output: the caller has to open every + /// matched file to hydrate line text, and grouping guarantees one open per + /// file. The response still carries `backend` and `status`, unlike + /// `search/line_matches_compact`, so callers can keep reporting which + /// backend served the query. + pub(crate) async fn search_grouped_line_matches( + &self, + request: SearchRequest, + ) -> Result { + self.send_repo_request( + "search/grouped_line_matches", + Request::SearchGroupedLineMatches { + params: SearchParams { + repo_id: self.repo_id.clone(), + query: request.query, + scope: request.scope, + }, + }, + |response| match response { + Response::SearchGroupedLineMatchesCompleted { + backend, + status, + results, + .. + } => Ok(GroupedLineMatchOutcome { + backend, + status, + results, + }), + other => unexpected_response("search/grouped_line_matches", other), + }, + None, + ) + .await + } + pub(crate) async fn glob(&self, request: GlobRequest) -> Result { self.send_repo_request( "glob", diff --git a/src/crates/services/services-integrations/src/workspace_search/flashgrep/mod.rs b/src/crates/services/services-integrations/src/workspace_search/flashgrep/mod.rs index 6ab28784e9..33df8d9c07 100644 --- a/src/crates/services/services-integrations/src/workspace_search/flashgrep/mod.rs +++ b/src/crates/services/services-integrations/src/workspace_search/flashgrep/mod.rs @@ -53,8 +53,8 @@ pub(crate) use protocol::{ pub(crate) use repo_session::FlashgrepRepoSession; pub(crate) use rpc_client::{drain_content_length_messages, ProtocolClient}; pub(crate) use types::{ - DirtyFileStats, FileCount, GlobOutcome, GlobRequest, OpenRepoParams, PathScope, QuerySpec, - RefreshPolicyConfig, RepoConfig, RepoPhase, RepoStatus, SearchBackend, SearchModeConfig, - SearchOutcome, SearchRequest, SearchResults, TaskKind, TaskPhase, TaskState, TaskStatus, - WorkspaceOverlayStatus, + DirtyFileStats, FileCount, GlobOutcome, GlobRequest, GroupedLineMatchResults, OpenRepoParams, + PathScope, QuerySpec, RefreshPolicyConfig, RepoConfig, RepoPhase, RepoStatus, SearchBackend, + SearchModeConfig, SearchOutcome, SearchRequest, SearchResults, TaskKind, TaskPhase, TaskState, + TaskStatus, WorkspaceOverlayStatus, }; diff --git a/src/crates/services/services-integrations/src/workspace_search/flashgrep/protocol.rs b/src/crates/services/services-integrations/src/workspace_search/flashgrep/protocol.rs index 7ad51afe0a..046f95c2bd 100644 --- a/src/crates/services/services-integrations/src/workspace_search/flashgrep/protocol.rs +++ b/src/crates/services/services-integrations/src/workspace_search/flashgrep/protocol.rs @@ -43,9 +43,16 @@ pub(crate) enum Request { GetRepoStatus { params: RepoRef, }, + RefreshRepo { + params: RefreshRepoParams, + }, Search { params: SearchParams, }, + #[serde(rename = "search/grouped_line_matches")] + SearchGroupedLineMatches { + params: SearchParams, + }, Glob { params: GlobParams, }, @@ -90,6 +97,15 @@ pub(crate) struct TaskRef { pub task_id: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct RefreshRepoParams { + pub repo_id: String, + /// `false` lets the daemon skip the walk when it already considers its view current; `true` + /// forces a reconcile regardless. + #[serde(default)] + pub force: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct OpenRepoParams { pub repo_path: PathBuf, @@ -107,8 +123,6 @@ pub(crate) struct SearchParams { pub query: QuerySpec, #[serde(default)] pub scope: PathScope, - #[serde(default)] - pub allow_scan_fallback: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -135,10 +149,6 @@ pub(crate) struct QuerySpec { pub word_regexp: bool, #[serde(default)] pub line_regexp: bool, - #[serde(default)] - pub before_context: usize, - #[serde(default)] - pub after_context: usize, #[serde(default = "default_top_k_tokens")] pub top_k_tokens: usize, #[serde(default)] @@ -198,8 +208,14 @@ impl Default for RepoConfig { #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct RefreshPolicyConfig { - #[serde(default = "default_rebuild_dirty_threshold")] - pub rebuild_dirty_threshold: usize, + #[serde(default = "default_base_delta_max_segments")] + pub base_delta_max_segments: usize, + #[serde(default = "default_base_delta_max_delete_segments")] + pub base_delta_max_delete_segments: usize, + #[serde(default = "default_base_delta_max_bytes_ratio")] + pub base_delta_max_bytes_ratio: f64, + #[serde(default = "default_base_head_cache_entries")] + pub base_head_cache_entries: usize, #[serde(default = "default_overlay_auto_checkpoint_max_uncommitted_ops")] pub overlay_auto_checkpoint_max_uncommitted_ops: u64, #[serde(default = "default_overlay_merge_min_delay_ms")] @@ -211,7 +227,10 @@ pub(crate) struct RefreshPolicyConfig { impl Default for RefreshPolicyConfig { fn default() -> Self { Self { - rebuild_dirty_threshold: default_rebuild_dirty_threshold(), + base_delta_max_segments: default_base_delta_max_segments(), + base_delta_max_delete_segments: default_base_delta_max_delete_segments(), + base_delta_max_bytes_ratio: default_base_delta_max_bytes_ratio(), + base_head_cache_entries: default_base_head_cache_entries(), overlay_auto_checkpoint_max_uncommitted_ops: default_overlay_auto_checkpoint_max_uncommitted_ops(), overlay_merge_min_delay_ms: default_overlay_merge_min_delay_ms(), @@ -315,6 +334,12 @@ pub(crate) enum Response { status: RepoStatus, results: SearchResults, }, + SearchGroupedLineMatchesCompleted { + repo_id: String, + backend: SearchBackend, + status: RepoStatus, + results: GroupedLineMatchResults, + }, GlobCompleted { repo_id: String, status: RepoStatus, @@ -362,13 +387,52 @@ pub struct RepoStatus { pub workspace_overlay_root: String, pub phase: RepoPhase, pub snapshot_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_head_commit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_head_commit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overlay_head_commit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overlay_base_manifest_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_manifest_id: Option, + #[serde(default)] + pub base_delta_depth: u32, + #[serde(default)] + pub base_delta_bytes: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_advance_target_head: Option, + #[serde(default)] + pub cached_head_count: usize, + #[serde(default)] + pub base_compaction_recommended: bool, pub last_probe_unix_secs: Option, pub last_rebuild_unix_secs: Option, pub dirty_files: DirtyFileStats, - pub rebuild_recommended: bool, pub active_task_id: Option, pub probe_healthy: bool, + /// `true` means the daemon still owes a worktree reconcile, so `dirty_files`, `phase` and the + /// published overlay describe the *last observed* worktree instead of the current one. Callers + /// that need authoritative state ask for it with `refresh_repo`. + /// + /// Defaulted so that any daemon older than v0.2.14 — which predates the field and never sends + /// it — still decodes: those builds reconcile synchronously inside `open_repo`, so `false` is + /// the correct reading for them. + #[serde(default)] + pub workspace_probe_pending: bool, pub last_error: Option, + /// Failure of the daemon's last background base-maintenance task (advance or compaction). + /// + /// Kept apart from `last_error` because that slot is shared with the worktree probe, and every + /// successful probe clears it — measured at ~3.5 s on v0.2.14, well inside our own 5 s idle + /// status poll, so a maintenance failure reported only there was gone before we could ever + /// render it. This slot is cleared only when the same kind of maintenance work succeeds. + /// + /// Defaulted so daemons older than v0.2.15, which never send the field, still decode as + /// "nothing failed" — for those builds the failure genuinely is unobservable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_maintenance_error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub overlay: Option, } @@ -394,6 +458,8 @@ pub struct DirtyFileStats { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WorkspaceOverlayStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_manifest_id: Option, pub committed_seq_no: u64, pub last_seq_no: u64, pub uncommitted_ops: u64, @@ -431,6 +497,8 @@ pub struct TaskStatus { pub enum TaskKind { BuildBaseSnapshot, RebuildBaseSnapshot, + AdvanceBaseSnapshot, + CompactBaseDeltas, RefreshWorkspace, } @@ -494,12 +562,48 @@ pub(crate) struct FileMatchCount { pub matched_occurrences: usize, } +/// A single matched line. +/// +/// The daemon reports positions only — there is no line text on the wire in any +/// search mode, so content previews have to be hydrated from disk by whoever can +/// read the files (see `workspace_search::line_hydration`). #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct LineMatch { pub path: String, pub line_number: usize, +} + +/// `search/grouped_line_matches` payload: one entry per file instead of one per +/// line, which is what makes single-open hydration possible. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct GroupedLineMatchResults { + pub candidate_docs: usize, + #[serde(default)] + pub searches_with_match: usize, + #[serde(default)] + pub bytes_searched: u64, + pub matched_lines: usize, + pub matched_occurrences: usize, #[serde(default)] - pub line_text: Option, + pub limit_reached: bool, + #[serde(default)] + pub summary: LineMatchSummary, + #[serde(default)] + pub files: Vec<(String, Vec)>, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub(crate) struct LineMatchSummary { + #[serde(default)] + pub files_with_matches: usize, + #[serde(default)] + pub top_files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LineMatchFileSummary { + pub path: String, + pub matched_lines: usize, } fn default_top_k_tokens() -> usize { @@ -518,12 +622,24 @@ fn default_max_sparse_len() -> usize { 8 } -fn default_rebuild_dirty_threshold() -> usize { - 256 +fn default_base_delta_max_segments() -> usize { + 8 +} + +fn default_base_delta_max_delete_segments() -> usize { + 8 +} + +fn default_base_delta_max_bytes_ratio() -> f64 { + 0.10 +} + +fn default_base_head_cache_entries() -> usize { + 4 } fn default_overlay_auto_checkpoint_max_uncommitted_ops() -> u64 { - 1_024 + 256 } fn default_overlay_merge_min_delay_ms() -> u64 { diff --git a/src/crates/services/services-integrations/src/workspace_search/flashgrep/rpc_client.rs b/src/crates/services/services-integrations/src/workspace_search/flashgrep/rpc_client.rs index ad4086ea3f..f755026cb6 100644 --- a/src/crates/services/services-integrations/src/workspace_search/flashgrep/rpc_client.rs +++ b/src/crates/services/services-integrations/src/workspace_search/flashgrep/rpc_client.rs @@ -313,7 +313,9 @@ fn request_name(request: &Request) -> &'static str { Request::TaskStatus { .. } => "task/status", Request::OpenRepo { .. } => "open_repo", Request::GetRepoStatus { .. } => "get_repo_status", + Request::RefreshRepo { .. } => "refresh_repo", Request::Search { .. } => "search", + Request::SearchGroupedLineMatches { .. } => "search/grouped_line_matches", Request::Glob { .. } => "glob", Request::CloseRepo { .. } => "close_repo", Request::Shutdown => "shutdown", diff --git a/src/crates/services/services-integrations/src/workspace_search/flashgrep/types.rs b/src/crates/services/services-integrations/src/workspace_search/flashgrep/types.rs index 8135e5a24a..acff457436 100644 --- a/src/crates/services/services-integrations/src/workspace_search/flashgrep/types.rs +++ b/src/crates/services/services-integrations/src/workspace_search/flashgrep/types.rs @@ -1,14 +1,13 @@ pub(crate) use super::protocol::{ - DirtyFileStats, FileCount, OpenRepoParams, PathScope, QuerySpec, RefreshPolicyConfig, - RepoConfig, RepoPhase, RepoStatus, SearchBackend, SearchModeConfig, SearchResults, TaskKind, - TaskPhase, TaskState, TaskStatus, WorkspaceOverlayStatus, + DirtyFileStats, FileCount, GroupedLineMatchResults, OpenRepoParams, PathScope, QuerySpec, + RefreshPolicyConfig, RepoConfig, RepoPhase, RepoStatus, SearchBackend, SearchModeConfig, + SearchResults, TaskKind, TaskPhase, TaskState, TaskStatus, WorkspaceOverlayStatus, }; #[derive(Debug, Clone)] pub(crate) struct SearchRequest { pub query: QuerySpec, pub scope: PathScope, - pub allow_scan_fallback: bool, } #[derive(Debug, Clone, Default)] @@ -23,6 +22,13 @@ pub(crate) struct SearchOutcome { pub results: SearchResults, } +#[derive(Debug, Clone)] +pub(crate) struct GroupedLineMatchOutcome { + pub backend: SearchBackend, + pub status: RepoStatus, + pub results: GroupedLineMatchResults, +} + #[derive(Debug, Clone)] pub(crate) struct GlobOutcome { pub status: RepoStatus, @@ -34,7 +40,6 @@ impl SearchRequest { Self { query, scope: PathScope::default(), - allow_scan_fallback: false, } } @@ -42,11 +47,6 @@ impl SearchRequest { self.scope = scope; self } - - pub(crate) fn with_scan_fallback(mut self, allow_scan_fallback: bool) -> Self { - self.allow_scan_fallback = allow_scan_fallback; - self - } } impl GlobRequest { diff --git a/src/crates/services/services-integrations/src/workspace_search/index_budget.rs b/src/crates/services/services-integrations/src/workspace_search/index_budget.rs new file mode 100644 index 0000000000..9dc570d5c8 --- /dev/null +++ b/src/crates/services/services-integrations/src/workspace_search/index_budget.rs @@ -0,0 +1,225 @@ +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +pub(crate) const DEFAULT_INDEX_DISK_BUDGET_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// Single definition of where a workspace's local flashgrep index lives. +/// +/// Budget maintenance and session creation must agree on this path, otherwise +/// maintenance silently stops finding the indexes it is meant to reclaim. +pub(crate) fn storage_root(repo_root: &Path) -> PathBuf { + repo_root + .join(".bitfun") + .join("search") + .join("flashgrep-index") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct IndexBudgetReport { + pub total_before: u64, + pub total_after: u64, + pub removed: Vec, + pub over_budget: bool, +} + +#[derive(Debug)] +struct IndexDirectory { + repo_root: PathBuf, + path: PathBuf, + bytes: u64, + modified: SystemTime, + recency_rank: usize, +} + +pub(crate) fn enforce( + workspace_roots: Vec, + protected_roots: HashSet, +) -> Result { + enforce_with_budget( + workspace_roots, + protected_roots, + DEFAULT_INDEX_DISK_BUDGET_BYTES, + ) +} + +pub(crate) fn remove_for_repo(repo_root: PathBuf) -> Result { + let path = storage_root(&repo_root); + match fs::metadata(&path) { + Ok(metadata) if metadata.is_dir() => { + fs::remove_dir_all(&path).map_err(|error| { + format!( + "failed to remove workspace search index {}: {error}", + path.display() + ) + })?; + Ok(true) + } + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(format!( + "failed to inspect workspace search index {}: {error}", + path.display() + )), + } +} + +fn enforce_with_budget( + workspace_roots: Vec, + protected_roots: HashSet, + budget_bytes: u64, +) -> Result { + let mut seen = HashSet::new(); + let mut indexes = Vec::new(); + let recency = workspace_roots + .iter() + .enumerate() + .map(|(rank, root)| (root.clone(), rank)) + .collect::>(); + + for repo_root in workspace_roots { + if !seen.insert(repo_root.clone()) { + continue; + } + let path = storage_root(&repo_root); + let Ok(metadata) = fs::metadata(&path) else { + continue; + }; + if !metadata.is_dir() { + continue; + } + let bytes = directory_size(&path)?; + indexes.push(IndexDirectory { + repo_root: repo_root.clone(), + path, + bytes, + modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), + recency_rank: recency.get(&repo_root).copied().unwrap_or(usize::MAX), + }); + } + + let total_before = indexes.iter().map(|index| index.bytes).sum::(); + let mut total_after = total_before; + let mut candidates = indexes; + candidates.sort_by(|left, right| { + right + .recency_rank + .cmp(&left.recency_rank) + .then_with(|| left.modified.cmp(&right.modified)) + }); + + let mut removed = Vec::new(); + for index in candidates { + if total_after <= budget_bytes { + break; + } + if protected_roots.contains(&index.repo_root) { + continue; + } + fs::remove_dir_all(&index.path).map_err(|error| { + format!( + "failed to remove workspace search index {}: {error}", + index.path.display() + ) + })?; + total_after = total_after.saturating_sub(index.bytes); + removed.push(index.repo_root); + } + + Ok(IndexBudgetReport { + total_before, + total_after, + removed, + over_budget: total_after > budget_bytes, + }) +} + +fn directory_size(path: &Path) -> Result { + let mut total = 0_u64; + for entry in fs::read_dir(path) + .map_err(|error| format!("failed to read index directory {}: {error}", path.display()))? + { + let entry = entry.map_err(|error| { + format!( + "failed to inspect index directory {}: {error}", + path.display() + ) + })?; + let file_type = entry.file_type().map_err(|error| { + format!( + "failed to inspect index entry {}: {error}", + entry.path().display() + ) + })?; + if file_type.is_dir() { + total = total.saturating_add(directory_size(&entry.path())?); + } else if file_type.is_file() { + total = total.saturating_add( + entry + .metadata() + .map_err(|error| { + format!( + "failed to read index entry metadata {}: {error}", + entry.path().display() + ) + })? + .len(), + ); + } + } + Ok(total) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::write; + use tempfile::TempDir; + + fn make_index(root: &Path, bytes: usize) { + let index = root.join(".bitfun/search/flashgrep-index"); + fs::create_dir_all(&index).expect("index directory should be created"); + write(index.join("payload"), vec![b'x'; bytes]).expect("index payload should be written"); + } + + #[test] + fn evicts_oldest_unprotected_indexes_until_budget_is_met() { + let temp = TempDir::new().expect("temp dir should be created"); + let newest = temp.path().join("newest"); + let older = temp.path().join("older"); + let oldest = temp.path().join("oldest"); + make_index(&newest, 4); + make_index(&older, 4); + make_index(&oldest, 4); + + let report = enforce_with_budget( + vec![newest.clone(), older.clone(), oldest.clone()], + HashSet::from([newest.clone()]), + 8, + ) + .expect("budget enforcement should succeed"); + + assert_eq!(report.total_before, 12); + assert_eq!(report.total_after, 8); + assert_eq!(report.removed, vec![oldest]); + assert!(newest.join(".bitfun/search/flashgrep-index").exists()); + assert!(older.join(".bitfun/search/flashgrep-index").exists()); + assert!(!report.over_budget); + } + + #[test] + fn reports_unavoidable_over_budget_when_all_indexes_are_protected() { + let temp = TempDir::new().expect("temp dir should be created"); + let root = temp.path().join("root"); + make_index(&root, 4); + + let report = enforce_with_budget(vec![root.clone()], HashSet::from([root]), 1) + .expect("budget enforcement should succeed"); + + assert_eq!(report.total_before, 4); + assert_eq!(report.total_after, 4); + assert!(report.over_budget); + assert!(report.removed.is_empty()); + } +} diff --git a/src/crates/services/services-integrations/src/workspace_search/index_queue.rs b/src/crates/services/services-integrations/src/workspace_search/index_queue.rs new file mode 100644 index 0000000000..1d6abcc78d --- /dev/null +++ b/src/crates/services/services-integrations/src/workspace_search/index_queue.rs @@ -0,0 +1,153 @@ +use std::collections::{HashSet, VecDeque}; +use std::path::PathBuf; + +use tokio::sync::Mutex; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkspaceSearchAutoIndexPriority { + Background, + Focused, +} + +#[derive(Debug, Default)] +struct QueueState { + pending: VecDeque, + queued: HashSet, + in_flight: HashSet, + driver_running: bool, +} + +#[derive(Debug, Default)] +pub(crate) struct AutoIndexQueue { + state: Mutex, +} + +impl AutoIndexQueue { + pub(crate) async fn enqueue( + &self, + repo_root: PathBuf, + priority: WorkspaceSearchAutoIndexPriority, + ) -> bool { + let mut state = self.state.lock().await; + if state.in_flight.contains(&repo_root) { + return false; + } + + if state.queued.contains(&repo_root) { + if priority == WorkspaceSearchAutoIndexPriority::Focused { + state.pending.retain(|path| path != &repo_root); + state.pending.push_front(repo_root); + } + return false; + } + + state.queued.insert(repo_root.clone()); + match priority { + WorkspaceSearchAutoIndexPriority::Background => state.pending.push_back(repo_root), + WorkspaceSearchAutoIndexPriority::Focused => state.pending.push_front(repo_root), + } + + if state.driver_running { + false + } else { + state.driver_running = true; + true + } + } + + pub(crate) async fn next(&self) -> Option { + let mut state = self.state.lock().await; + let repo_root = state.pending.pop_front(); + if let Some(repo_root) = repo_root.as_ref() { + state.queued.remove(repo_root); + state.in_flight.insert(repo_root.clone()); + } else { + state.driver_running = false; + } + repo_root + } + + pub(crate) async fn complete(&self, repo_root: &PathBuf) { + self.state.lock().await.in_flight.remove(repo_root); + } + + pub(crate) async fn protected_roots(&self) -> Vec { + let state = self.state.lock().await; + state + .queued + .iter() + .chain(state.in_flight.iter()) + .cloned() + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn focused_workspaces_jump_ahead_of_background_items() { + let queue = AutoIndexQueue::default(); + assert!( + queue + .enqueue( + PathBuf::from("background-a"), + WorkspaceSearchAutoIndexPriority::Background, + ) + .await + ); + assert!( + !queue + .enqueue( + PathBuf::from("background-b"), + WorkspaceSearchAutoIndexPriority::Background, + ) + .await + ); + assert!( + !queue + .enqueue( + PathBuf::from("focused"), + WorkspaceSearchAutoIndexPriority::Focused, + ) + .await + ); + + assert_eq!(queue.next().await, Some(PathBuf::from("focused"))); + queue.complete(&PathBuf::from("focused")).await; + assert_eq!(queue.next().await, Some(PathBuf::from("background-a"))); + } + + #[tokio::test] + async fn duplicate_items_are_deduplicated_and_in_flight_items_are_ignored() { + let queue = AutoIndexQueue::default(); + assert!( + queue + .enqueue( + PathBuf::from("repo"), + WorkspaceSearchAutoIndexPriority::Background, + ) + .await + ); + assert!( + !queue + .enqueue( + PathBuf::from("repo"), + WorkspaceSearchAutoIndexPriority::Background, + ) + .await + ); + assert_eq!(queue.next().await, Some(PathBuf::from("repo"))); + assert!( + !queue + .enqueue( + PathBuf::from("repo"), + WorkspaceSearchAutoIndexPriority::Focused, + ) + .await + ); + queue.complete(&PathBuf::from("repo")).await; + assert_eq!(queue.next().await, None); + } +} diff --git a/src/crates/services/services-integrations/src/workspace_search/line_hydration.rs b/src/crates/services/services-integrations/src/workspace_search/line_hydration.rs new file mode 100644 index 0000000000..59a027f37f --- /dev/null +++ b/src/crates/services/services-integrations/src/workspace_search/line_hydration.rs @@ -0,0 +1,340 @@ +//! Hydration of daemon line matches with on-disk line text. +//! +//! The flashgrep daemon reports match *positions* only — every search mode +//! returns `{path, line_number}` and never the line itself. Content output +//! therefore has to read the matched files locally; this module does that with +//! exactly one open per file by consuming the per-file grouping from +//! `search/grouped_line_matches`. +//! +//! Callers must apply their result limit *before* hydration (that is what +//! `max_results` here is for): reading files for matches that are about to be +//! truncated away is pure I/O waste on large result sets. + +use bitfun_services_core::filesystem::{ + ContentMatchPreviewBuilder, FileSearchResult, SearchMatchType, +}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +/// Mirrors `tool_execution::search::grep_search::MAX_DISPLAY_COLUMNS` so an +/// indexed content search renders a long line the same way the ripgrep fallback +/// does. +pub(crate) const MAX_HYDRATED_LINE_COLUMNS: usize = 500; +const TRUNCATION_SUFFIX: &str = " [truncated]"; + +#[derive(Debug, Default)] +pub(crate) struct HydratedLineMatches { + pub results: Vec, + /// Matched lines the daemon reported that `max_results` dropped before any + /// file was opened. + pub dropped_lines: usize, + /// Files that could not be read, i.e. removed or made unreadable since the + /// snapshot the daemon answered from. Their matches are still returned as + /// path + line number, without text. + pub unreadable_files: usize, +} + +/// Applies `max_results` to the daemon's per-file grouping *before* any line is +/// read, and normalises each file's line numbers to sorted-unique order. +/// +/// Split out from hydration itself because the remote path needs the same +/// budgeting decision — which files to open, and how many matches were dropped +/// on the floor — before it can build the batch it ships over SSH. +pub(crate) fn plan_wanted_lines( + files: &[(String, Vec)], + max_results: Option, +) -> (Vec<(String, Vec)>, usize) { + let mut planned = Vec::with_capacity(files.len()); + let mut dropped_lines = 0usize; + let mut remaining = max_results.unwrap_or(usize::MAX); + + for (path, line_numbers) in files { + let mut wanted = line_numbers.clone(); + wanted.sort_unstable(); + wanted.dedup(); + + if remaining == 0 { + dropped_lines += wanted.len(); + continue; + } + if wanted.len() > remaining { + dropped_lines += wanted.len() - remaining; + wanted.truncate(remaining); + } + remaining -= wanted.len(); + planned.push((path.clone(), wanted)); + } + + (planned, dropped_lines) +} + +/// Turns one file's wanted line numbers plus their (possibly missing) text into +/// content search results, appending them to `out`. +/// +/// `texts` is positional: one entry per `wanted` line, `None` when the line +/// could not be read. Shared with the remote path so an indexed match renders +/// identically whether its text came off the local disk or off an SSH batch. +pub(crate) fn push_line_results( + path: &str, + wanted: &[usize], + texts: Vec>, + preview: Option<&ContentMatchPreviewBuilder>, + out: &mut Vec, +) { + let file_name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(path) + .to_string(); + + for (line_number, text) in wanted.iter().copied().zip(texts) { + let (preview_before, preview_inside, preview_after) = match (preview, text.as_deref()) { + (Some(preview), Some(text)) => preview.preview(text), + _ => (None, None, None), + }; + + out.push(FileSearchResult { + path: path.to_string(), + name: file_name.clone(), + is_directory: false, + match_type: SearchMatchType::Content, + line_number: Some(line_number), + matched_content: text.map(|text| clamp_display_line(&text)), + preview_before, + preview_inside, + preview_after, + }); + } +} + +/// Reads the matched lines for `files` (daemon order preserved) and turns them +/// into content search results. +/// +/// This performs blocking I/O; run it on a blocking thread. +pub(crate) fn hydrate_grouped_line_matches( + files: &[(String, Vec)], + max_results: Option, + preview: Option<&ContentMatchPreviewBuilder>, +) -> HydratedLineMatches { + let (planned, dropped_lines) = plan_wanted_lines(files, max_results); + let mut outcome = HydratedLineMatches { + dropped_lines, + ..HydratedLineMatches::default() + }; + + for (path, wanted) in &planned { + let texts = match read_wanted_lines(Path::new(path), wanted) { + Some(texts) => texts, + None => { + outcome.unreadable_files += 1; + vec![None; wanted.len()] + } + }; + + push_line_results(path, wanted, texts, preview, &mut outcome.results); + } + + outcome +} + +/// Reads the requested 1-based line numbers in a single pass. +/// +/// Returns `None` when the file cannot be opened at all; otherwise one entry per +/// requested line, `None` for line numbers past end of file (the snapshot can be +/// slightly ahead of the worktree). +fn read_wanted_lines(path: &Path, wanted: &[usize]) -> Option>> { + let last_wanted = wanted.last().copied()?; + let file = File::open(path).ok()?; + let reader = BufReader::new(file); + + let mut texts = vec![None; wanted.len()]; + let mut next_wanted = 0usize; + + for (index, line) in reader.split(b'\n').enumerate() { + let line_number = index + 1; + let Ok(line) = line else { + // Unreadable mid-file (I/O error, not a decoding issue): keep what + // has been hydrated so far rather than dropping the whole file. + break; + }; + + while next_wanted < wanted.len() && wanted[next_wanted] < line_number { + next_wanted += 1; + } + if next_wanted >= wanted.len() { + break; + } + if wanted[next_wanted] == line_number { + let text = String::from_utf8_lossy(&line); + texts[next_wanted] = Some(text.trim_end_matches('\r').to_string()); + next_wanted += 1; + } + if line_number >= last_wanted { + break; + } + } + + Some(texts) +} + +pub(crate) fn clamp_display_line(line: &str) -> String { + if line.chars().count() <= MAX_HYDRATED_LINE_COLUMNS { + return line.to_string(); + } + + let head = line + .chars() + .take(MAX_HYDRATED_LINE_COLUMNS) + .collect::(); + format!("{head}{TRUNCATION_SUFFIX}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + fn preview_builder(pattern: &str) -> ContentMatchPreviewBuilder { + ContentMatchPreviewBuilder::new(pattern, true, false, false).expect("preview builder") + } + + #[test] + fn hydrates_matched_lines_in_file_order() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("sample.txt"); + fs::write(&path, "alpha\nneedle one\ngamma\nneedle two\n").expect("write"); + let path = path.to_string_lossy().to_string(); + + let outcome = hydrate_grouped_line_matches( + &[(path.clone(), vec![4, 2])], + None, + Some(&preview_builder("needle")), + ); + + assert_eq!(outcome.dropped_lines, 0); + assert_eq!(outcome.unreadable_files, 0); + let rendered = outcome + .results + .iter() + .map(|result| { + ( + result.line_number, + result.matched_content.clone(), + result.preview_inside.clone(), + ) + }) + .collect::>(); + assert_eq!( + rendered, + vec![ + ( + Some(2), + Some("needle one".to_string()), + Some("needle".to_string()) + ), + ( + Some(4), + Some("needle two".to_string()), + Some("needle".to_string()) + ), + ] + ); + } + + #[test] + fn strips_carriage_returns_and_keeps_multibyte_text() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("crlf.txt"); + fs::write(&path, "第一行 needle\r\n第二行\r\n").expect("write"); + let path = path.to_string_lossy().to_string(); + + let outcome = hydrate_grouped_line_matches( + &[(path, vec![1])], + None, + Some(&preview_builder("needle")), + ); + + let result = &outcome.results[0]; + assert_eq!(result.matched_content.as_deref(), Some("第一行 needle")); + assert_eq!(result.preview_before.as_deref(), Some("第一行 ")); + assert_eq!(result.preview_inside.as_deref(), Some("needle")); + } + + #[test] + fn line_past_end_of_file_yields_no_text() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("short.txt"); + fs::write(&path, "only line\n").expect("write"); + let path = path.to_string_lossy().to_string(); + + let outcome = hydrate_grouped_line_matches(&[(path, vec![1, 9])], None, None); + + assert_eq!(outcome.results.len(), 2); + assert_eq!( + outcome.results[0].matched_content.as_deref(), + Some("only line") + ); + assert_eq!(outcome.results[1].line_number, Some(9)); + assert_eq!(outcome.results[1].matched_content, None); + assert_eq!(outcome.unreadable_files, 0); + } + + #[test] + fn unreadable_file_still_reports_positions() { + let dir = tempdir().expect("tempdir"); + let missing = dir.path().join("gone.txt").to_string_lossy().to_string(); + + let outcome = hydrate_grouped_line_matches(&[(missing.clone(), vec![3])], None, None); + + assert_eq!(outcome.unreadable_files, 1); + assert_eq!(outcome.results.len(), 1); + assert_eq!(outcome.results[0].path, missing); + assert_eq!(outcome.results[0].line_number, Some(3)); + assert_eq!(outcome.results[0].matched_content, None); + } + + #[test] + fn max_results_is_applied_before_reading_files() { + let dir = tempdir().expect("tempdir"); + let first = dir.path().join("first.txt"); + fs::write(&first, "needle a\nneedle b\n").expect("write"); + let missing = dir.path().join("never-opened.txt"); + + let outcome = hydrate_grouped_line_matches( + &[ + (first.to_string_lossy().to_string(), vec![1, 2]), + (missing.to_string_lossy().to_string(), vec![1, 2, 3]), + ], + Some(1), + None, + ); + + assert_eq!(outcome.results.len(), 1); + assert_eq!(outcome.dropped_lines, 4); + // The second file was never opened, so it is not counted as unreadable. + assert_eq!(outcome.unreadable_files, 0); + } + + #[test] + fn very_long_lines_are_clamped_like_the_ripgrep_path() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("long.txt"); + let long_line = format!("needle{}", "x".repeat(MAX_HYDRATED_LINE_COLUMNS * 2)); + fs::write(&path, format!("{long_line}\n")).expect("write"); + let path = path.to_string_lossy().to_string(); + + let outcome = hydrate_grouped_line_matches(&[(path, vec![1])], None, None); + + let content = outcome.results[0] + .matched_content + .as_deref() + .expect("content"); + assert!(content.ends_with(TRUNCATION_SUFFIX)); + assert_eq!( + content.chars().count(), + MAX_HYDRATED_LINE_COLUMNS + TRUNCATION_SUFFIX.chars().count() + ); + } +} diff --git a/src/crates/services/services-integrations/src/workspace_search/mod.rs b/src/crates/services/services-integrations/src/workspace_search/mod.rs index 40c33c33aa..2a6d4541e6 100644 --- a/src/crates/services/services-integrations/src/workspace_search/mod.rs +++ b/src/crates/services/services-integrations/src/workspace_search/mod.rs @@ -4,11 +4,16 @@ //! indexed-search DTOs. Product/runtime crates may wrap it to provide product //! config, bootstrap hooks, and legacy error mapping. +mod auto_index; pub(crate) mod flashgrep; +mod index_budget; +mod index_queue; +pub(crate) mod line_hydration; pub(crate) mod result_mapping; mod service; mod types; +pub use index_queue::WorkspaceSearchAutoIndexPriority; pub use service::{ resolve_workspace_search_daemon_program_path, workspace_search_daemon_available, workspace_search_daemon_binary_name, workspace_search_daemon_binary_names, @@ -17,10 +22,10 @@ pub use service::{ }; pub use types::{ ContentSearchOutputMode, ContentSearchRequest, ContentSearchResult, GlobSearchRequest, - GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchBackend, - WorkspaceSearchContextLine, WorkspaceSearchDirtyFiles, WorkspaceSearchFileCount, - WorkspaceSearchHit, WorkspaceSearchLine, WorkspaceSearchMatch, WorkspaceSearchMatchLocation, - WorkspaceSearchOverlayStatus, WorkspaceSearchRepoPhase, WorkspaceSearchRepoStatus, - WorkspaceSearchTaskKind, WorkspaceSearchTaskPhase, WorkspaceSearchTaskState, - WorkspaceSearchTaskStatus, + GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchAutoIndexDecision, + WorkspaceSearchAutoIndexStatus, WorkspaceSearchBackend, WorkspaceSearchContextLine, + WorkspaceSearchDirtyFiles, WorkspaceSearchFileCount, WorkspaceSearchHit, WorkspaceSearchLine, + WorkspaceSearchMatch, WorkspaceSearchMatchLocation, WorkspaceSearchOverlayStatus, + WorkspaceSearchRepoPhase, WorkspaceSearchRepoStatus, WorkspaceSearchTaskKind, + WorkspaceSearchTaskPhase, WorkspaceSearchTaskState, WorkspaceSearchTaskStatus, }; diff --git a/src/crates/services/services-integrations/src/workspace_search/result_mapping.rs b/src/crates/services/services-integrations/src/workspace_search/result_mapping.rs index 4c939f4d9b..130c0ae7f9 100644 --- a/src/crates/services/services-integrations/src/workspace_search/result_mapping.rs +++ b/src/crates/services/services-integrations/src/workspace_search/result_mapping.rs @@ -33,44 +33,34 @@ pub(crate) fn convert_search_results( } } +/// Maps `search` line matches to results carrying positions only. +/// +/// The daemon never sends line text on the wire, so callers that cannot read the +/// matched files themselves (the remote SSH transport) surface path + line number +/// without content. Local searches use `search/grouped_line_matches` plus +/// `line_hydration` instead, which fills in the text and the previews. fn convert_hits_to_file_search_results(search_results: &SearchResults) -> Vec { search_results .line_matches .iter() - .map(|matched| { - let matched_content = matched - .line_text - .clone() - .unwrap_or_else(|| format!("line {}", matched.line_number)); - let (preview_before, preview_inside, preview_after) = matched - .line_text - .as_deref() - .map(split_preview) - .unwrap_or((None, None, None)); - - FileSearchResult { - path: matched.path.clone(), - name: Path::new(&matched.path) - .file_name() - .and_then(|file_name| file_name.to_str()) - .unwrap_or(&matched.path) - .to_string(), - is_directory: false, - match_type: SearchMatchType::Content, - line_number: Some(matched.line_number), - matched_content: Some(matched_content), - preview_before, - preview_inside, - preview_after, - } + .map(|matched| FileSearchResult { + path: matched.path.clone(), + name: Path::new(&matched.path) + .file_name() + .and_then(|file_name| file_name.to_str()) + .unwrap_or(&matched.path) + .to_string(), + is_directory: false, + match_type: SearchMatchType::Content, + line_number: Some(matched.line_number), + matched_content: None, + preview_before: None, + preview_inside: None, + preview_after: None, }) .collect() } -fn split_preview(line_text: &str) -> (Option, Option, Option) { - (None, Some(line_text.to_string()), None) -} - fn convert_file_counts_to_search_results(search_results: &SearchResults) -> Vec { search_results .file_counts diff --git a/src/crates/services/services-integrations/src/workspace_search/service.rs b/src/crates/services/services-integrations/src/workspace_search/service.rs index 8b1b6345bf..42c4802f5d 100644 --- a/src/crates/services/services-integrations/src/workspace_search/service.rs +++ b/src/crates/services/services-integrations/src/workspace_search/service.rs @@ -3,7 +3,7 @@ use super::flashgrep::{ RefreshPolicyConfig, RepoConfig, RepoSession, SearchRequest, FLASHGREP_LOG_TARGET, }; use async_trait::async_trait; -use bitfun_services_core::filesystem::FileSearchOutcome; +use bitfun_services_core::filesystem::{ContentMatchPreviewBuilder, FileSearchOutcome}; use std::collections::{HashMap, HashSet}; use std::ffi::OsString; use std::path::{Component, Path, PathBuf}; @@ -14,10 +14,17 @@ use std::sync::{ use std::time::{Duration, Instant}; use tokio::sync::{Mutex, RwLock}; +use super::line_hydration::{hydrate_grouped_line_matches, HydratedLineMatches}; use super::result_mapping::convert_search_results; use super::types::{ - ContentSearchRequest, ContentSearchResult, GlobSearchRequest, GlobSearchResult, - IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchFileCount, + ContentSearchOutputMode, ContentSearchRequest, ContentSearchResult, GlobSearchRequest, + GlobSearchResult, IndexTaskHandle, WorkspaceIndexStatus, WorkspaceSearchAutoIndexDecision, + WorkspaceSearchAutoIndexStatus, WorkspaceSearchFileCount, +}; +use super::{ + auto_index::{self, AutoIndexDecision, AutoIndexPolicy}, + index_budget, + index_queue::{AutoIndexQueue, WorkspaceSearchAutoIndexPriority}, }; pub type WorkspaceSearchResult = Result; @@ -77,6 +84,14 @@ pub struct WorkspaceSearchService { client: ManagedClient, sessions: RwLock>, open_guards: Mutex>>>, + auto_index_queue: Arc, + /// Last auto-index policy outcome per workspace root. The daemon reports `needs_index` + /// whether the policy is still evaluating or has declined, so the reason is recorded here + /// and surfaced with the status instead of only being logged. + auto_index_decisions: RwLock>, + /// Workspace roots last handed to budget maintenance, kept so that a build + /// finishing later can re-enforce the budget against the same set. + index_budget_roots: RwLock>, session_idle_grace: Duration, hooks: Arc, } @@ -109,6 +124,9 @@ impl WorkspaceSearchService { client, sessions: RwLock::new(HashMap::new()), open_guards: Mutex::new(HashMap::new()), + auto_index_queue: Arc::new(AutoIndexQueue::default()), + auto_index_decisions: RwLock::new(HashMap::new()), + index_budget_roots: RwLock::new(Vec::new()), session_idle_grace: DEFAULT_SESSION_IDLE_GRACE, hooks, } @@ -119,7 +137,9 @@ impl WorkspaceSearchService { repo_root: impl AsRef, ) -> WorkspaceSearchResult { let session = self.get_or_open_session(repo_root.as_ref()).await?; - self.index_status_for_session(session).await + let mut status = self.index_status_for_session(session).await?; + status.auto_index = Some(self.auto_index_status(repo_root.as_ref()).await); + Ok(status) } pub async fn get_index_status( @@ -127,7 +147,40 @@ impl WorkspaceSearchService { repo_root: impl AsRef, ) -> WorkspaceSearchResult { let session = self.get_or_open_session(repo_root.as_ref()).await?; - self.index_status_for_session(session).await + let mut status = self.index_status_for_session(session).await?; + status.auto_index = Some(self.auto_index_status(repo_root.as_ref()).await); + Ok(status) + } + + /// Reports the recorded auto-index decision, defaulting to `Pending` for a workspace the + /// policy has not reached yet (the queue runs asynchronously behind workspace activation). + async fn auto_index_status(&self, repo_root: &Path) -> WorkspaceSearchAutoIndexStatus { + let pending = WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::Pending, + threshold: auto_index::DEFAULT_AUTO_INDEX_MIN_FILES, + indexable_files: None, + reason: None, + }; + let Ok(repo_root) = normalize_repo_root(repo_root) else { + return pending; + }; + self.auto_index_decisions + .read() + .await + .get(&repo_root) + .cloned() + .unwrap_or(pending) + } + + async fn record_auto_index_decision( + &self, + repo_root: &Path, + status: WorkspaceSearchAutoIndexStatus, + ) { + self.auto_index_decisions + .write() + .await + .insert(repo_root.to_path_buf(), status); } pub async fn build_index( @@ -200,6 +253,12 @@ impl WorkspaceSearchService { let scope_globs_count = scope.globs.len(); let scope_types_count = scope.types.len(); let max_results = request.max_results.filter(|limit| *limit > 0); + let preview_spec = ContentPreviewSpec { + pattern: request.pattern.clone(), + case_sensitive: request.case_sensitive, + use_regex: request.use_regex, + whole_word: request.whole_word, + }; let query = QuerySpec { pattern: request.pattern, patterns: Vec::new(), @@ -209,8 +268,6 @@ impl WorkspaceSearchService { fixed_strings: !request.use_regex, word_regexp: request.whole_word, line_regexp: false, - before_context: request.before_context, - after_context: request.after_context, top_k_tokens: DEFAULT_TOP_K_TOKENS, max_count: None, global_max_results: max_results, @@ -219,45 +276,92 @@ impl WorkspaceSearchService { let session = self.get_or_open_session(&repo_root).await?; let session_ready_at = Instant::now(); - let search = FlashgrepRepoSession::search( - session.as_ref(), - SearchRequest::new(query) - .with_scope(scope) - .with_scan_fallback(true), - ) - .await - .map_err(map_flashgrep_error("Content search failed"))?; - let search_completed_at = Instant::now(); - - let mut results = convert_search_results(&search.results, request.output_mode); - let converted_at = Instant::now(); - let truncated = max_results - .map(|limit| results.len() >= limit) - .unwrap_or(false); - if let Some(limit) = max_results { - results.truncate(limit); - } + let search_request = SearchRequest::new(query).with_scope(scope); + + let (result, search_completed_at, converted_at) = match request.output_mode { + // Content output needs the line text, which no daemon search mode + // returns, so it takes the grouped variant and hydrates from disk. + ContentSearchOutputMode::Content => { + let grouped = session + .search_grouped_line_matches(search_request) + .await + .map_err(map_flashgrep_error("Content search failed"))?; + let search_completed_at = Instant::now(); + + let hydrated = hydrate_grouped_content_matches( + grouped.results.files, + max_results, + preview_spec, + ) + .await?; + let converted_at = Instant::now(); - let result = ContentSearchResult { - outcome: FileSearchOutcome { results, truncated }, - file_counts: search - .results - .file_counts - .clone() - .into_iter() - .map(WorkspaceSearchFileCount::from) - .collect(), - hits: Vec::new(), - backend: search.backend.into(), - repo_status: search.status.into(), - candidate_docs: search.results.candidate_docs, - matched_lines: search.results.matched_lines, - matched_occurrences: search.results.matched_occurrences, + if hydrated.unreadable_files > 0 { + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Workspace content search could not read {} matched file(s); reporting their matches without line text: repo_root={}", + hydrated.unreadable_files, + repo_root.display() + ); + } + + let truncated = grouped.results.limit_reached || hydrated.dropped_lines > 0; + let result = ContentSearchResult { + outcome: FileSearchOutcome { + results: hydrated.results, + truncated, + }, + // `search/grouped_line_matches` reports per-file counts only + // for its top-10 summary, which is not the full list the + // count output mode promises, so it is left to that mode. + file_counts: Vec::new(), + hits: Vec::new(), + backend: grouped.backend.into(), + repo_status: grouped.status.into(), + candidate_docs: grouped.results.candidate_docs, + matched_lines: grouped.results.matched_lines, + matched_occurrences: grouped.results.matched_occurrences, + }; + (result, search_completed_at, converted_at) + } + ContentSearchOutputMode::Count | ContentSearchOutputMode::FilesWithMatches => { + let search = FlashgrepRepoSession::search(session.as_ref(), search_request) + .await + .map_err(map_flashgrep_error("Content search failed"))?; + let search_completed_at = Instant::now(); + + let mut results = convert_search_results(&search.results, request.output_mode); + let converted_at = Instant::now(); + let truncated = max_results + .map(|limit| results.len() >= limit) + .unwrap_or(false); + if let Some(limit) = max_results { + results.truncate(limit); + } + + let result = ContentSearchResult { + outcome: FileSearchOutcome { results, truncated }, + file_counts: search + .results + .file_counts + .clone() + .into_iter() + .map(WorkspaceSearchFileCount::from) + .collect(), + hits: Vec::new(), + backend: search.backend.into(), + repo_status: search.status.into(), + candidate_docs: search.results.candidate_docs, + matched_lines: search.results.matched_lines, + matched_occurrences: search.results.matched_occurrences, + }; + (result, search_completed_at, converted_at) + } }; log::debug!( target: FLASHGREP_LOG_TARGET, - "Workspace content search completed: repo_root={}, pattern={}, output_mode={:?}, search_mode={:?}, scope_roots={}, globs={}, file_types={}, max_results={:?}, backend={:?}, repo_phase={:?}, rebuild_recommended={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, returned_results={}, truncated={}, normalize_ms={}, build_scope_ms={}, session_ms={}, search_ms={}, convert_ms={}, total_ms={}", + "Workspace content search completed: repo_root={}, pattern={}, output_mode={:?}, search_mode={:?}, scope_roots={}, globs={}, file_types={}, max_results={:?}, backend={:?}, repo_phase={:?}, base_advance_in_progress={}, workspace_probe_pending={}, dirty_modified={}, dirty_deleted={}, dirty_new={}, candidate_docs={}, matched_lines={}, matched_occurrences={}, returned_results={}, truncated={}, normalize_ms={}, build_scope_ms={}, session_ms={}, search_ms={}, convert_ms={}, total_ms={}", repo_root.display(), pattern_for_log, request.output_mode, @@ -268,7 +372,8 @@ impl WorkspaceSearchService { max_results, result.backend, result.repo_status.phase, - result.repo_status.rebuild_recommended, + result.repo_status.base_advance_in_progress, + result.repo_status.workspace_probe_pending, result.repo_status.dirty_files.modified, result.repo_status.dirty_files.deleted, result.repo_status.dirty_files.new, @@ -353,6 +458,152 @@ impl WorkspaceSearchService { }); } + /// Queue an automatic base-index build without blocking workspace activation or search. + pub async fn schedule_auto_index( + self: &Arc, + repo_root: impl AsRef, + priority: WorkspaceSearchAutoIndexPriority, + ) { + let Ok(repo_root) = normalize_repo_root(repo_root.as_ref()) else { + return; + }; + if self.auto_index_queue.enqueue(repo_root, priority).await { + let service = Arc::clone(self); + tokio::spawn(async move { + service.run_auto_index_queue().await; + }); + } + } + + /// Reclaim old local workspace indexes without touching active or queued workspaces. + pub async fn enforce_index_disk_budget( + &self, + workspace_roots: Vec, + protected_roots: Vec, + ) { + let workspace_roots = workspace_roots + .into_iter() + .filter_map(|path| normalize_repo_root(&path).ok()) + .collect::>(); + let protected = protected_roots + .into_iter() + .filter_map(|path| normalize_repo_root(&path).ok()) + .collect::>(); + *self.index_budget_roots.write().await = workspace_roots.clone(); + self.enforce_normalized_index_disk_budget(workspace_roots, protected) + .await; + } + + /// Re-run budget maintenance against the roots cached by the last + /// `enforce_index_disk_budget` call. + /// + /// Automatic index builds are the only thing that grows on-disk index size, + /// so they are the trigger; a workspace switch on its own changes nothing + /// and would only pay for a recursive size walk of every known index. + async fn enforce_cached_index_disk_budget(&self) { + let workspace_roots = self.index_budget_roots.read().await.clone(); + if workspace_roots.is_empty() { + return; + } + // Caller-supplied protection (the focused workspace) is not cached: it + // already holds an open session, and live sessions are unioned below. + self.enforce_normalized_index_disk_budget(workspace_roots, HashSet::new()) + .await; + } + + async fn enforce_normalized_index_disk_budget( + &self, + workspace_roots: Vec, + protected_roots: HashSet, + ) { + let mut protected = protected_roots; + protected.extend(self.sessions.read().await.keys().cloned()); + protected.extend(self.auto_index_queue.protected_roots().await); + + let result = + tokio::task::spawn_blocking(move || index_budget::enforce(workspace_roots, protected)) + .await; + match result { + Ok(Ok(report)) => { + if report.removed.is_empty() && !report.over_budget { + return; + } + log::info!( + target: FLASHGREP_LOG_TARGET, + "Workspace search index budget maintenance completed: total_before_bytes={}, total_after_bytes={}, removed={}, over_budget={}", + report.total_before, + report.total_after, + report.removed.len(), + report.over_budget + ); + } + Ok(Err(error)) => log::warn!( + target: FLASHGREP_LOG_TARGET, + "Workspace search index budget maintenance failed: {}", + error + ), + Err(error) => log::warn!( + target: FLASHGREP_LOG_TARGET, + "Workspace search index budget worker failed: {}", + error + ), + } + } + + pub async fn remove_workspace_index(&self, repo_root: impl AsRef) { + let Ok(repo_root) = normalize_repo_root(repo_root.as_ref()) else { + return; + }; + if self + .auto_index_queue + .protected_roots() + .await + .into_iter() + .any(|path| path == repo_root) + { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Keeping workspace search index because automatic indexing is still active: path={}", + repo_root.display() + ); + return; + } + + let session = self.sessions.write().await.remove(&repo_root); + self.open_guards.lock().await.remove(&repo_root); + if let Some(entry) = session { + if let Err(error) = entry.session.close().await { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Failed to close workspace search session before index removal: path={}, error={}", + repo_root.display(), + error + ); + } + } + + let removal_root = repo_root.clone(); + match tokio::task::spawn_blocking(move || index_budget::remove_for_repo(removal_root)).await + { + Ok(Ok(true)) => log::info!( + target: FLASHGREP_LOG_TARGET, + "Removed workspace search index: path={}", + repo_root.display() + ), + Ok(Ok(false)) => {} + Ok(Err(error)) => log::warn!( + target: FLASHGREP_LOG_TARGET, + "Failed to remove workspace search index: {}", + error + ), + Err(error) => log::warn!( + target: FLASHGREP_LOG_TARGET, + "Workspace search index removal worker failed: {}", + error + ), + } + } + pub async fn shutdown_all_daemons(&self) { let released_sessions = self.sessions.write().await.drain().count(); self.open_guards.lock().await.clear(); @@ -510,6 +761,117 @@ impl WorkspaceSearchService { .clone()) } + async fn run_auto_index_queue(self: Arc) { + let mut built_any = false; + while let Some(repo_root) = self.auto_index_queue.next().await { + match self.auto_index_repo(&repo_root).await { + Ok(built) => built_any |= built, + Err(error) => log::warn!( + target: FLASHGREP_LOG_TARGET, + "Automatic workspace search indexing failed: path={}, error={}", + repo_root.display(), + error + ), + } + self.auto_index_queue.complete(&repo_root).await; + } + + // Enforce only after the queue drains: the roots still queued would be + // protected anyway, and one walk covers every build in this drain. + if built_any { + self.enforce_cached_index_disk_budget().await; + } + } + + /// Returns whether this call actually produced index data on disk. + async fn auto_index_repo(&self, repo_root: &Path) -> WorkspaceSearchResult { + let repo_config = self.hooks.repo_config().await; + let decision = auto_index::evaluate( + repo_root.to_path_buf(), + AutoIndexPolicy { + min_indexable_files: auto_index::DEFAULT_AUTO_INDEX_MIN_FILES, + max_file_size: repo_config.max_file_size, + }, + ) + .await; + match decision { + AutoIndexDecision::BelowThreshold { indexable_files } => { + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Skipping automatic workspace search indexing below file threshold: path={}, indexable_files={}, threshold={}", + repo_root.display(), + indexable_files, + auto_index::DEFAULT_AUTO_INDEX_MIN_FILES + ); + self.record_auto_index_decision( + repo_root, + WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::BelowThreshold, + threshold: auto_index::DEFAULT_AUTO_INDEX_MIN_FILES, + indexable_files: Some(indexable_files), + reason: None, + }, + ) + .await; + return Ok(false); + } + AutoIndexDecision::Unsupported { reason } => { + log::warn!( + target: FLASHGREP_LOG_TARGET, + "Skipping automatic workspace search indexing because flashgrep requires a Git worktree: path={}, reason={}", + repo_root.display(), + reason + ); + self.record_auto_index_decision( + repo_root, + WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::Unsupported, + threshold: auto_index::DEFAULT_AUTO_INDEX_MIN_FILES, + indexable_files: None, + reason: Some(reason), + }, + ) + .await; + return Ok(false); + } + AutoIndexDecision::Eligible { + indexable_files_at_least, + } => { + log::info!( + target: FLASHGREP_LOG_TARGET, + "Starting automatic workspace search indexing: path={}, indexable_files_at_least={}, threshold={}", + repo_root.display(), + indexable_files_at_least, + auto_index::DEFAULT_AUTO_INDEX_MIN_FILES + ); + self.record_auto_index_decision( + repo_root, + WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::Eligible, + threshold: auto_index::DEFAULT_AUTO_INDEX_MIN_FILES, + indexable_files: Some(indexable_files_at_least), + reason: None, + }, + ) + .await; + } + } + + let session = self.get_or_open_session(repo_root).await?; + let status = wait_for_auto_index_repo_status(session.as_ref()).await?; + if !repo_needs_auto_index_build(status.phase, status.active_task_id.as_deref()) { + return Ok(false); + } + + let task = FlashgrepRepoSession::build_index(session.as_ref()) + .await + .map_err(map_flashgrep_error( + "Failed to start automatic workspace search indexing", + ))?; + wait_for_index_task(session.as_ref(), task.task_id).await?; + Ok(true) + } + async fn index_status_for_session( &self, session: Arc, @@ -539,10 +901,11 @@ impl WorkspaceSearchService { Ok(WorkspaceIndexStatus { repo_status: repo_status.into(), active_task: active_task.map(Into::into), + auto_index: None, }) } - async fn release_repo_if_idle(&self, repo_root: PathBuf) { + async fn release_repo_if_idle(self: &Arc, repo_root: PathBuf) { let Some(expected_epoch) = self .sessions .read() @@ -553,6 +916,24 @@ impl WorkspaceSearchService { return; }; + let active_session = self + .sessions + .read() + .await + .get(&repo_root) + .map(|entry| entry.session.clone()); + if let Some(session) = active_session { + if session + .status() + .await + .map(|status| status.active_task_id.is_some()) + .unwrap_or(false) + { + self.schedule_repo_release(repo_root); + return; + } + } + let entry = { let mut sessions = self.sessions.write().await; let Some(entry) = sessions.get(&repo_root) else { @@ -583,6 +964,102 @@ impl WorkspaceSearchService { } } +/// The pattern and flags needed to recompute match highlighting locally. +/// +/// The daemon reports neither the matched text nor its offsets, so previews are +/// rebuilt client-side from the same pattern that was searched. +#[derive(Debug, Clone)] +struct ContentPreviewSpec { + pattern: String, + case_sensitive: bool, + use_regex: bool, + whole_word: bool, +} + +/// Reads matched line text off disk for the grouped daemon result. +/// +/// `max_results` is applied inside the hydration pass so files whose matches are +/// about to be dropped are never opened. +async fn hydrate_grouped_content_matches( + files: Vec<(String, Vec)>, + max_results: Option, + preview_spec: ContentPreviewSpec, +) -> WorkspaceSearchResult { + tokio::task::spawn_blocking(move || { + // A pattern the daemon accepts can still fail to compile here (different + // regex dialect, or a multiline query that no single line matches), in + // which case results keep their line text and lose only the highlight. + let preview = match ContentMatchPreviewBuilder::new( + &preview_spec.pattern, + preview_spec.case_sensitive, + preview_spec.use_regex, + preview_spec.whole_word, + ) { + Ok(preview) => Some(preview), + Err(error) => { + log::debug!( + target: FLASHGREP_LOG_TARGET, + "Workspace content search preview highlighting disabled: {error}" + ); + None + } + }; + hydrate_grouped_line_matches(&files, max_results, preview.as_ref()) + }) + .await + .map_err(|error| format!("Content search line hydration failed: {error}")) +} + +async fn wait_for_index_task(session: &RepoSession, task_id: String) -> WorkspaceSearchResult<()> { + loop { + let task = session + .task_status(task_id.clone()) + .await + .map_err(map_flashgrep_error( + "Failed to poll automatic workspace search indexing", + ))?; + match task.state { + super::flashgrep::TaskState::Queued | super::flashgrep::TaskState::Running => { + tokio::time::sleep(Duration::from_millis(500)).await; + } + super::flashgrep::TaskState::Completed => return Ok(()), + super::flashgrep::TaskState::Failed => { + return Err(task.error.unwrap_or(task.message)); + } + super::flashgrep::TaskState::Cancelled => { + return Err("Automatic workspace search indexing was cancelled".to_string()); + } + } + } +} + +async fn wait_for_auto_index_repo_status( + session: &RepoSession, +) -> WorkspaceSearchResult { + loop { + let status = session.status().await.map_err(map_flashgrep_error( + "Failed to fetch repository status for automatic indexing", + ))?; + if !matches!(status.phase, super::flashgrep::RepoPhase::Opening) { + return Ok(status); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn repo_needs_auto_index_build( + phase: super::flashgrep::RepoPhase, + active_task_id: Option<&str>, +) -> bool { + active_task_id.is_none() + && matches!( + phase, + super::flashgrep::RepoPhase::MissingBaseSnapshot + | super::flashgrep::RepoPhase::BuildingBaseSnapshot + | super::flashgrep::RepoPhase::RebuildingBaseSnapshot + ) +} + impl Default for WorkspaceSearchService { fn default() -> Self { Self::new() @@ -734,10 +1211,7 @@ fn push_exe_relative_bundle_candidates( } fn default_storage_root(repo_root: &Path) -> PathBuf { - repo_root - .join(".bitfun") - .join("search") - .join("flashgrep-index") + index_budget::storage_root(repo_root) } fn abbreviate_pattern_for_log(pattern: &str) -> String { @@ -940,7 +1414,7 @@ fn map_flashgrep_error( mod tests { use super::*; use crate::workspace_search::flashgrep::SearchResults; - use crate::workspace_search::ContentSearchOutputMode; + use crate::workspace_search::{ContentSearchOutputMode, WorkspaceSearchRepoStatus}; fn empty_search_results() -> SearchResults { serde_json::from_value(serde_json::json!({ @@ -953,6 +1427,88 @@ mod tests { .expect("empty search results should decode with defaulted collections") } + fn repo_status_json(extra_status_fields: serde_json::Value) -> serde_json::Value { + let mut status = serde_json::json!({ + "repo_id": "/repo", + "repo_path": "/repo", + "storage_root": "/repo/.bitfun/search/flashgrep-index", + "base_snapshot_root": "/repo/.bitfun/search/flashgrep-index/base-snapshot", + "workspace_overlay_root": "/repo/.bitfun/search/flashgrep-index/workspace-overlay", + "phase": "ready_clean", + "snapshot_key": "base-git:abc123+cfg:deadbeef", + "last_probe_unix_secs": null, + "last_rebuild_unix_secs": null, + "dirty_files": {"modified": 0, "deleted": 0, "new": 0}, + "active_task_id": null, + "probe_healthy": true, + "last_error": null + }); + let object = status.as_object_mut().expect("status should be an object"); + for (key, value) in extra_status_fields + .as_object() + .expect("extra fields should be an object") + { + object.insert(key.clone(), value.clone()); + } + status + } + + #[test] + fn repo_status_reports_no_pending_probe_for_daemons_that_omit_the_field() { + // Daemons older than v0.2.14 predate `workspace_probe_pending` and reconcile the worktree + // inside `open_repo`, so a status without the field describes the current worktree and must + // decode as "nothing owed" rather than failing. + let status: crate::workspace_search::flashgrep::RepoStatus = + serde_json::from_value(repo_status_json(serde_json::json!({}))) + .expect("a status without the field should decode"); + assert!(!status.workspace_probe_pending); + let exposed: WorkspaceSearchRepoStatus = status.into(); + assert!(!exposed.workspace_probe_pending); + } + + #[test] + fn repo_status_carries_a_pending_probe_through_to_callers() { + let status: crate::workspace_search::flashgrep::RepoStatus = serde_json::from_value( + repo_status_json(serde_json::json!({"workspace_probe_pending": true})), + ) + .expect("a status with the field should decode"); + assert!(status.workspace_probe_pending); + // Callers decide what to do about staleness, so the flag has to survive the conversion into + // the type the UI and agent tools actually read. + let exposed: WorkspaceSearchRepoStatus = status.into(); + assert!(exposed.workspace_probe_pending); + } + + #[test] + fn repo_status_reports_no_maintenance_error_for_daemons_that_omit_the_field() { + // Daemons older than v0.2.15 have no separate maintenance slot at all: a failed compaction + // lands in `last_error`, which the next successful worktree probe clears within a few + // seconds. Decoding the absent field as "nothing failed" is honest for those builds — the + // failure really is unobservable there — and must not fail the decode. + let status: crate::workspace_search::flashgrep::RepoStatus = + serde_json::from_value(repo_status_json(serde_json::json!({}))) + .expect("a status without the field should decode"); + assert!(status.last_maintenance_error.is_none()); + let exposed: WorkspaceSearchRepoStatus = status.into(); + assert!(exposed.last_maintenance_error.is_none()); + } + + #[test] + fn repo_status_carries_a_maintenance_error_through_to_callers() { + let status: crate::workspace_search::flashgrep::RepoStatus = + serde_json::from_value(repo_status_json(serde_json::json!({ + "last_maintenance_error": "io error: Permission denied (os error 13)", + }))) + .expect("a status with the field should decode"); + // The whole point of the separate slot is that it reaches a status poller, so it has to + // survive the conversion into the type the UI actually renders. + let exposed: WorkspaceSearchRepoStatus = status.into(); + assert_eq!( + exposed.last_maintenance_error.as_deref(), + Some("io error: Permission denied (os error 13)") + ); + } + #[test] fn content_search_output_modes_use_current_flashgrep_protocol_modes() { assert_eq!( @@ -969,6 +1525,26 @@ mod tests { ); } + #[test] + fn automatic_indexing_only_starts_when_a_base_snapshot_is_missing_and_idle() { + assert!(repo_needs_auto_index_build( + crate::workspace_search::flashgrep::RepoPhase::MissingBaseSnapshot, + None, + )); + assert!(repo_needs_auto_index_build( + crate::workspace_search::flashgrep::RepoPhase::BuildingBaseSnapshot, + None, + )); + assert!(!repo_needs_auto_index_build( + crate::workspace_search::flashgrep::RepoPhase::MissingBaseSnapshot, + Some("task-1"), + )); + assert!(!repo_needs_auto_index_build( + crate::workspace_search::flashgrep::RepoPhase::ReadyClean, + None, + )); + } + #[test] fn glob_scope_preprocessing_extracts_static_pattern_prefix() { let repo_root = std::env::temp_dir().join("bitfun-workspace-search-test-repo"); @@ -1004,7 +1580,10 @@ mod tests { } #[test] - fn content_search_converts_line_matches_without_line_text() { + fn plain_line_matches_carry_positions_without_content() { + // `search` never returns line text in any mode, so this mapping (used by + // the remote transport) must not invent a placeholder; local searches go + // through `search/grouped_line_matches` + `line_hydration` instead. let mut search_results = empty_search_results(); search_results.line_matches = serde_json::from_value(serde_json::json!([{ "path": "src/search.rs", @@ -1018,32 +1597,57 @@ mod tests { assert_eq!(results[0].path, "src/search.rs"); assert_eq!(results[0].name, "search.rs"); assert_eq!(results[0].line_number, Some(42)); - assert_eq!(results[0].matched_content.as_deref(), Some("line 42")); + assert_eq!(results[0].matched_content, None); assert_eq!(results[0].preview_inside, None); } #[test] - fn content_search_converts_line_matches_with_line_text_preview() { - let mut search_results = empty_search_results(); - search_results.line_matches = serde_json::from_value(serde_json::json!([{ - "path": "src/search.rs", - "line_number": 42, - "line_text": "let result = search();" - }])) - .expect("line_matches should decode"); - - let results = convert_search_results(&search_results, ContentSearchOutputMode::Content); - - assert_eq!(results.len(), 1); - assert_eq!( - results[0].matched_content.as_deref(), - Some("let result = search();") - ); - assert_eq!( - results[0].preview_inside.as_deref(), - Some("let result = search();") - ); - assert_eq!(results[0].preview_before, None); - assert_eq!(results[0].preview_after, None); + fn grouped_line_match_results_decode_with_backend_and_status() { + // The compact variant is deliberately not implemented: its response + // omits repo_id/backend/status, which every caller here reports. + let response: super::super::flashgrep::Response = serde_json::from_value( + serde_json::json!({ + "kind": "search_grouped_line_matches_completed", + "repo_id": "/repo", + "backend": "indexed_clean", + "status": { + "repo_id": "/repo", + "repo_path": "/repo", + "storage_root": "/repo/.bitfun/search/flashgrep-index", + "base_snapshot_root": "/repo/.bitfun/search/flashgrep-index/base-snapshot", + "workspace_overlay_root": "/repo/.bitfun/search/flashgrep-index/workspace-overlay", + "phase": "ready_clean", + "snapshot_key": null, + "last_probe_unix_secs": null, + "last_rebuild_unix_secs": null, + "dirty_files": {"modified": 0, "deleted": 0, "new": 0}, + "active_task_id": null, + "probe_healthy": true, + "last_error": null + }, + "results": { + "candidate_docs": 3, + "matched_lines": 2, + "matched_occurrences": 2, + "files": [["/repo/src/search.rs", [42, 43]]] + } + }), + ) + .expect("grouped response should decode"); + + match response { + super::super::flashgrep::Response::SearchGroupedLineMatchesCompleted { + results, + .. + } => { + assert_eq!(results.matched_lines, 2); + assert_eq!( + results.files, + vec![("/repo/src/search.rs".to_string(), vec![42, 43])] + ); + assert!(!results.limit_reached); + } + other => panic!("unexpected response: {other:?}"), + } } } diff --git a/src/crates/services/services-integrations/src/workspace_search/types.rs b/src/crates/services/services-integrations/src/workspace_search/types.rs index af91fcc713..12e21cc451 100644 --- a/src/crates/services/services-integrations/src/workspace_search/types.rs +++ b/src/crates/services/services-integrations/src/workspace_search/types.rs @@ -37,8 +37,9 @@ pub struct ContentSearchRequest { pub use_regex: bool, pub whole_word: bool, pub multiline: bool, - pub before_context: usize, - pub after_context: usize, + // No context-line fields: the daemon has no context-line support, so a request that asks for + // them is routed to ripgrep before it ever becomes a `ContentSearchRequest`. Carrying them + // here is what let `-A`/`-B`/`-C` be silently dropped on the wire once before. pub max_results: Option, pub globs: Vec, pub file_types: Vec, @@ -104,6 +105,8 @@ impl From for WorkspaceSearchRepoPhase { pub enum WorkspaceSearchTaskKind { Build, Rebuild, + Advance, + Compact, Refresh, } @@ -112,6 +115,8 @@ impl From for WorkspaceSearchTaskKind { match value { FlashgrepTaskKind::BuildBaseSnapshot => Self::Build, FlashgrepTaskKind::RebuildBaseSnapshot => Self::Rebuild, + FlashgrepTaskKind::AdvanceBaseSnapshot => Self::Advance, + FlashgrepTaskKind::CompactBaseDeltas => Self::Compact, FlashgrepTaskKind::RefreshWorkspace => Self::Refresh, } } @@ -182,6 +187,7 @@ impl From for WorkspaceSearchDirtyFiles { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct WorkspaceSearchOverlayStatus { + pub base_manifest_id: Option, pub committed_seq_no: u64, pub last_seq_no: u64, pub uncommitted_ops: u64, @@ -199,6 +205,7 @@ pub struct WorkspaceSearchOverlayStatus { impl From for WorkspaceSearchOverlayStatus { fn from(value: FlashgrepWorkspaceOverlayStatus) -> Self { Self { + base_manifest_id: value.base_manifest_id, committed_seq_no: value.committed_seq_no, last_seq_no: value.last_seq_no, uncommitted_ops: value.uncommitted_ops, @@ -225,13 +232,26 @@ pub struct WorkspaceSearchRepoStatus { pub workspace_overlay_root: String, pub phase: WorkspaceSearchRepoPhase, pub snapshot_key: Option, + pub base_head_commit: Option, + pub workspace_head_commit: Option, + /// True while the daemon is advancing the base snapshot toward a newer commit. + pub base_advance_in_progress: bool, + pub base_advance_target_head: Option, + pub base_delta_depth: u32, + pub base_compaction_recommended: bool, pub last_probe_unix_secs: Option, pub last_rebuild_unix_secs: Option, pub dirty_files: WorkspaceSearchDirtyFiles, - pub rebuild_recommended: bool, pub active_task_id: Option, pub probe_healthy: bool, + /// True while the daemon still owes a worktree reconcile, which makes `dirty_files` and `phase` + /// the last observed state rather than the current one. Daemons that reconcile synchronously at + /// open never report it. + pub workspace_probe_pending: bool, pub last_error: Option, + /// Failure of the daemon's last background base-maintenance task. Unlike `last_error`, this is + /// not cleared by a successful worktree probe, so status polling can actually observe it. + pub last_maintenance_error: Option, pub overlay: Option, } @@ -245,13 +265,20 @@ impl From for WorkspaceSearchRepoStatus { workspace_overlay_root: value.workspace_overlay_root, phase: value.phase.into(), snapshot_key: value.snapshot_key, + base_head_commit: value.base_head_commit, + workspace_head_commit: value.workspace_head_commit, + base_advance_in_progress: value.base_advance_target_head.is_some(), + base_advance_target_head: value.base_advance_target_head, + base_delta_depth: value.base_delta_depth, + base_compaction_recommended: value.base_compaction_recommended, last_probe_unix_secs: value.last_probe_unix_secs, last_rebuild_unix_secs: value.last_rebuild_unix_secs, dirty_files: value.dirty_files.into(), - rebuild_recommended: value.rebuild_recommended, active_task_id: value.active_task_id, probe_healthy: value.probe_healthy, + workspace_probe_pending: value.workspace_probe_pending, last_error: value.last_error, + last_maintenance_error: value.last_maintenance_error, overlay: value.overlay.map(Into::into), } } @@ -349,11 +376,47 @@ pub struct WorkspaceSearchHit { pub lines: Vec, } +/// Outcome of BitFun's automatic-index policy for a workspace. +/// +/// flashgrep reports `needs_index` both while the policy is still evaluating a workspace and +/// after it has deliberately declined to build one, so the daemon's phase alone cannot explain +/// to a user why nothing is happening. This carries that BitFun-side decision to the UI. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum WorkspaceSearchAutoIndexDecision { + /// The policy has not evaluated this workspace yet. + Pending, + /// The workspace qualifies; an index build was started. + Eligible, + /// The workspace is too small to be worth indexing; fallback search stays in use. + BelowThreshold, + /// The workspace cannot be indexed at all (for example, it is not a Git worktree). + Unsupported, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceSearchAutoIndexStatus { + pub decision: WorkspaceSearchAutoIndexDecision, + /// Indexable-file count the policy requires before it builds an index. + pub threshold: usize, + /// Exact for `BelowThreshold`. For `Eligible` the count stops as soon as the threshold is + /// reached, so it is a lower bound there. `None` for `Pending` and `Unsupported`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub indexable_files: Option, + /// Set for `Unsupported` only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct WorkspaceIndexStatus { pub repo_status: WorkspaceSearchRepoStatus, pub active_task: Option, + /// Absent on transports that have no BitFun-side auto-index policy (remote SSH workspaces). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auto_index: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -385,3 +448,51 @@ pub struct IndexTaskHandle { pub task: WorkspaceSearchTaskStatus, pub repo_status: WorkspaceSearchRepoStatus, } + +#[cfg(test)] +mod tests { + use super::*; + + /// The frontend reads these names directly, and it distinguishes "no count available" from + /// "zero files", so absent optionals must stay absent rather than serialize as `null`. + #[test] + fn auto_index_status_serializes_in_the_shape_the_ui_expects() { + let below = WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::BelowThreshold, + threshold: 2_000, + indexable_files: Some(216), + reason: None, + }; + let value = serde_json::to_value(&below).expect("serialize"); + assert_eq!(value["decision"], "belowThreshold"); + assert_eq!(value["threshold"], 2_000); + assert_eq!(value["indexableFiles"], 216); + assert!(value.get("reason").is_none()); + + let pending = WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::Pending, + threshold: 2_000, + indexable_files: None, + reason: None, + }; + let value = serde_json::to_value(&pending).expect("serialize"); + assert_eq!(value["decision"], "pending"); + assert!(value.get("indexableFiles").is_none()); + } + + /// `Unsupported` is the only decision that carries a reason, and it has no file count to + /// report; the UI interpolates the reason straight into its sentence. + #[test] + fn unsupported_carries_a_reason_and_no_count() { + let value = serde_json::to_value(WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision::Unsupported, + threshold: 2_000, + indexable_files: None, + reason: Some("not a Git worktree".to_string()), + }) + .expect("serialize"); + assert_eq!(value["decision"], "unsupported"); + assert_eq!(value["reason"], "not a Git worktree"); + assert!(value.get("indexableFiles").is_none()); + } +} diff --git a/src/crates/services/services-integrations/tests/workspace_search_contracts.rs b/src/crates/services/services-integrations/tests/workspace_search_contracts.rs index 10d45f8087..c9ba2e4f63 100644 --- a/src/crates/services/services-integrations/tests/workspace_search_contracts.rs +++ b/src/crates/services/services-integrations/tests/workspace_search_contracts.rs @@ -1,8 +1,13 @@ #![cfg(feature = "workspace-search")] +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; + use bitfun_services_integrations::workspace_search::{ - workspace_search_daemon_binary_name, workspace_search_daemon_binary_names, - workspace_search_daemon_missing_hint, WorkspaceSearchService, + resolve_workspace_search_daemon_program_path, workspace_search_daemon_binary_name, + workspace_search_daemon_binary_names, workspace_search_daemon_missing_hint, + ContentSearchOutputMode, ContentSearchRequest, WorkspaceSearchService, }; #[test] @@ -26,3 +31,110 @@ fn daemon_missing_hint_preserves_env_override_guidance() { fn service_constructs_without_core_runtime_dependencies() { let _service = WorkspaceSearchService::new(); } + +/// Live end-to-end check that content matches carry real line text. +/// +/// The flashgrep daemon returns match positions only, so content output is +/// hydrated from disk. This test spawns the real daemon, so it is ignored by +/// default; run it with +/// `cargo test -p bitfun-services-integrations --features workspace-search +/// --test workspace_search_contracts -- --ignored`. +#[tokio::test] +#[ignore = "spawns the real flashgrep daemon and indexes a temporary repository"] +async fn content_search_hydrates_real_line_text_and_previews() { + let Some(daemon) = resolve_workspace_search_daemon_program_path() else { + panic!( + "flashgrep daemon binary not found: {}", + workspace_search_daemon_missing_hint() + ); + }; + println!("using daemon: {}", daemon.display()); + + let repo = tempfile::tempdir().expect("temp repo should be created"); + let repo_root = repo.path().canonicalize().expect("repo root canonicalizes"); + git(&repo_root, &["init"]); + git(&repo_root, &["config", "user.email", "test@example.com"]); + git(&repo_root, &["config", "user.name", "test"]); + std::fs::write( + repo_root.join("needle.rs"), + "fn main() {}\nlet answer = compute_needle_value(41 + 1);\n", + ) + .expect("fixture file should be written"); + git(&repo_root, &["add", "."]); + git(&repo_root, &["commit", "-m", "fixture"]); + + let service = WorkspaceSearchService::new(); + let handle = service + .build_index(&repo_root) + .await + .expect("index build should start"); + println!("index task: {:?}", handle.task.state); + + let deadline = Instant::now() + Duration::from_secs(120); + while Instant::now() < deadline { + let status = service + .get_index_status(&repo_root) + .await + .expect("index status should be readable"); + if status.active_task.is_none() { + break; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + + let result = service + .search_content(ContentSearchRequest { + repo_root: repo_root.clone(), + search_path: None, + pattern: "compute_needle_value".to_string(), + output_mode: ContentSearchOutputMode::Content, + case_sensitive: false, + use_regex: false, + whole_word: false, + multiline: false, + max_results: None, + globs: Vec::new(), + file_types: Vec::new(), + exclude_file_types: Vec::new(), + }) + .await + .expect("content search should succeed"); + + println!( + "backend={:?} results={:?}", + result.backend, result.outcome.results + ); + let matched = result + .outcome + .results + .first() + .expect("content search should return the fixture match"); + assert_eq!(matched.line_number, Some(2)); + assert_eq!( + matched.matched_content.as_deref(), + Some("let answer = compute_needle_value(41 + 1);") + ); + assert_eq!( + matched.preview_inside.as_deref(), + Some("compute_needle_value") + ); + assert_eq!(matched.preview_before.as_deref(), Some("let answer = ")); + assert_eq!(matched.preview_after.as_deref(), Some("(41 + 1);")); + + service.remove_workspace_index(&repo_root).await; + service.shutdown_all_daemons().await; +} + +#[cfg(test)] +fn git(repo_root: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repo_root) + .args(args) + .output() + .expect("git should be available"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx b/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx index 9e7cd08972..4c86ffacf0 100644 --- a/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx +++ b/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx @@ -65,13 +65,6 @@ interface WorkspaceItemProps { onDragEnd?: React.DragEventHandler; } -function getIndexActionKind(phase?: string | null): 'build' | 'rebuild' { - if (!phase || phase === 'needs_index' || phase === 'preparing') { - return 'build'; - } - return 'rebuild'; -} - const WorkspaceItem: React.FC = ({ workspace, isActive, @@ -241,11 +234,15 @@ const WorkspaceItem: React.FC = ({ const repoStatus = workspaceSearchIndex.indexStatus?.repoStatus ?? null; const activeTask = workspaceSearchIndex.indexStatus?.activeTask ?? null; + // A non-Git workspace can never be indexed; that is a property of the folder, not a fault, + // so it stays on the neutral gray tone and gets its own wording instead of a red "unhealthy". + const isNonGitWorkspace = workspaceSearchIndex.unsupportedReason === 'non_git'; const phase = repoStatus?.phase; const isTaskActive = activeTask?.state === 'queued' || activeTask?.state === 'running'; const hasError = Boolean( workspaceSearchIndex.error || repoStatus?.lastError + || repoStatus?.lastMaintenanceError || activeTask?.error || activeTask?.state === 'failed' ); @@ -263,24 +260,50 @@ const WorkspaceItem: React.FC = ({ || phase === 'preparing' || phase === 'building' || phase === 'refreshing' - || Boolean(repoStatus?.rebuildRecommended) + || Boolean(repoStatus?.baseAdvanceInProgress) ) { tone = 'yellow'; } else if (phase === 'ready' || phase === 'tracking_changes') { tone = 'green'; } - const phaseLabel = tFiles(`search.index.phase.${phase ?? 'unknown'}`, { - defaultValue: phase ?? tFiles('search.index.phase.unknown'), - }); + // The daemon says `needs_index` both while BitFun's auto-index policy is still evaluating the + // workspace and after it deliberately declined, so without the policy's own decision the UI + // can only hedge. When the decision is known it replaces the hedged wording with the reason. + const autoIndex = workspaceSearchIndex.indexStatus?.autoIndex ?? null; + const autoIndexExplanation = + phase === 'needs_index' && !isTaskActive && autoIndex + ? tFiles(`search.index.autoIndex.${autoIndex.decision}`, { + defaultValue: '', + files: autoIndex.indexableFiles ?? 0, + threshold: autoIndex.threshold, + reason: autoIndex.reason ?? '', + }) || null + : null; + const isBelowThreshold = autoIndex?.decision === 'belowThreshold'; + + const phaseLabel = isNonGitWorkspace + ? tFiles('search.index.phase.non_git') + : isBelowThreshold && phase === 'needs_index' + ? tFiles('search.index.phase.no_index_needed') + : tFiles(`search.index.phase.${phase ?? 'unknown'}`, { + defaultValue: phase ?? tFiles('search.index.phase.unknown'), + }); const title = tFiles(`search.index.indicator.tones.${tone}`); - const summary = repoStatus - ? tFiles(`search.index.summary.${phase ?? 'unavailable'}`, { - defaultValue: tFiles('search.index.summary.unavailable'), - }) - : workspaceSearchIndex.loading - ? tFiles('search.index.indicator.checking') - : tFiles('search.index.summary.unavailable'); + let summary: string; + if (isNonGitWorkspace) { + summary = tFiles('search.index.summary.non_git'); + } else if (autoIndexExplanation) { + summary = autoIndexExplanation; + } else if (repoStatus) { + summary = tFiles(`search.index.summary.${phase ?? 'unavailable'}`, { + defaultValue: tFiles('search.index.summary.unavailable'), + }); + } else if (workspaceSearchIndex.loading) { + summary = tFiles('search.index.indicator.checking'); + } else { + summary = tFiles('search.index.summary.unavailable'); + } const activeTaskLabel = activeTask ? tFiles(`search.index.taskState.${activeTask.state}`, { defaultValue: activeTask.state, @@ -312,7 +335,15 @@ const WorkspaceItem: React.FC = ({ new: repoStatus.dirtyFiles.new, }) : null; - const errorText = workspaceSearchIndex.error ?? activeTask?.error ?? repoStatus?.lastError ?? null; + // `lastError` is cleared by every successful worktree probe, so it is usually already gone by + // the time we poll; `lastMaintenanceError` is the slot a background compaction/advance failure + // survives in, and is the only one that reliably reaches this render. + const errorText = + workspaceSearchIndex.error + ?? activeTask?.error + ?? repoStatus?.lastError + ?? repoStatus?.lastMaintenanceError + ?? null; return { tone, @@ -325,8 +356,11 @@ const WorkspaceItem: React.FC = ({ progressPercent, progressPercentLabel, dirtyFilesLabel, - rebuildRecommended: Boolean(repoStatus?.rebuildRecommended), + baseAdvanceInProgress: Boolean(repoStatus?.baseAdvanceInProgress), probeHealthy: repoStatus?.probeHealthy ?? true, + // Not a degradation: the daemon owes a worktree reconcile, so the dirty counts above are from + // a moment ago. It clears itself, which is why it stays a tooltip note and not a badge. + workspaceProbePending: Boolean(repoStatus?.workspaceProbePending), errorText, ariaLabel: `${tFiles('search.index.indicator.label')}: ${title} · ${phaseLabel}`, }; @@ -336,35 +370,28 @@ const WorkspaceItem: React.FC = ({ workspaceSearchIndex.error, workspaceSearchIndex.indexStatus, workspaceSearchIndex.loading, + workspaceSearchIndex.unsupportedReason, ]); - const searchIndexActionKind = getIndexActionKind( - workspaceSearchIndex.indexStatus?.repoStatus.phase ?? null - ); - const searchIndexActionLabel = tFiles( - searchIndexActionKind === 'build' - ? 'search.index.actions.build' - : 'search.index.actions.rebuild' + const searchIndexPhase = workspaceSearchIndex.indexStatus?.repoStatus.phase ?? null; + const canRebuildSearchIndex = Boolean( + searchIndexPhase + && searchIndexPhase !== 'needs_index' + && searchIndexPhase !== 'preparing' + && searchIndexPhase !== 'building' ); const handleSearchIndexAction = useCallback(async () => { - const result = - searchIndexActionKind === 'build' - ? await workspaceSearchIndex.buildIndex() - : await workspaceSearchIndex.rebuildIndex(); + const result = await workspaceSearchIndex.rebuildIndex(); if (!result) { return; } notificationService.success( - tFiles( - searchIndexActionKind === 'build' - ? 'notifications.searchIndexBuildStarted' - : 'notifications.searchIndexRebuildStarted' - ), + tFiles('notifications.searchIndexRebuildStarted'), { duration: 2200 } ); - }, [searchIndexActionKind, tFiles, workspaceSearchIndex]); + }, [tFiles, workspaceSearchIndex]); const updateMenuPosition = useCallback(() => { const anchor = menuAnchorRef.current; @@ -1214,9 +1241,14 @@ const WorkspaceItem: React.FC = ({ {searchIndexIndicator.dirtyFilesLabel} ) : null} - {searchIndexIndicator.rebuildRecommended ? ( + {searchIndexIndicator.workspaceProbePending ? ( +
+ {tFiles('search.index.indicator.probePending')} +
+ ) : null} + {searchIndexIndicator.baseAdvanceInProgress ? (
- {tFiles('search.index.indicator.rebuildRecommended')} + {tFiles('search.index.indicator.baseAdvancing')}
) : null} {!searchIndexIndicator.probeHealthy ? ( @@ -1229,24 +1261,26 @@ const WorkspaceItem: React.FC = ({ {searchIndexIndicator.errorText} ) : null} -
- -
+ {canRebuildSearchIndex ? ( +
+ +
+ ) : null} diff --git a/src/web-ui/src/app/components/panels/FilesPanel.tsx b/src/web-ui/src/app/components/panels/FilesPanel.tsx index bcf2050fbd..a445df25ba 100644 --- a/src/web-ui/src/app/components/panels/FilesPanel.tsx +++ b/src/web-ui/src/app/components/panels/FilesPanel.tsx @@ -1162,9 +1162,16 @@ const FilesPanel: React.FC = ({ defaultValue: contentSearchMetadata.repoPhase, })} - {contentSearchMetadata.rebuildRecommended ? ( + {contentSearchMetadata.baseAdvanceInProgress ? ( - {t('search.index.badges.rebuildRecommended')} + {t('search.index.badges.baseAdvancing')} + + ) : null} + {contentSearchMetadata.workspaceProbePending ? ( + // Neutral, not warning: the reconcile clears itself and the results are still + // usable — they just describe the worktree from a moment ago. + + {t('search.index.badges.probePending')} ) : null} diff --git a/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts b/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts index 12e4be55cf..cc303caadf 100644 --- a/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts @@ -50,6 +50,12 @@ interface WorkspaceSearchRepoStatusRaw { workspaceOverlayRoot: string; phase: WorkspaceSearchIndexStatus['repoStatus']['phase']; snapshotKey?: string | null; + baseHeadCommit?: string | null; + workspaceHeadCommit?: string | null; + baseAdvanceInProgress: boolean; + baseAdvanceTargetHead?: string | null; + baseDeltaDepth: number; + baseCompactionRecommended: boolean; lastProbeUnixSecs?: number | null; lastRebuildUnixSecs?: number | null; dirtyFiles: { @@ -57,10 +63,11 @@ interface WorkspaceSearchRepoStatusRaw { deleted: number; new: number; }; - rebuildRecommended: boolean; activeTaskId?: string | null; probeHealthy: boolean; + workspaceProbePending?: boolean; lastError?: string | null; + lastMaintenanceError?: string | null; overlay?: WorkspaceSearchIndexStatus['repoStatus']['overlay'] | null; } @@ -80,9 +87,17 @@ interface WorkspaceSearchTaskStatusRaw { error?: string | null; } +interface WorkspaceSearchAutoIndexStatusRaw { + decision: NonNullable['decision']; + threshold: number; + indexableFiles?: number | null; + reason?: string | null; +} + interface WorkspaceSearchIndexStatusRaw { repoStatus: WorkspaceSearchRepoStatusRaw; activeTask?: WorkspaceSearchTaskStatusRaw | null; + autoIndex?: WorkspaceSearchAutoIndexStatusRaw | null; } interface WorkspaceSearchIndexTaskHandleRaw { @@ -125,13 +140,20 @@ function mapWorkspaceSearchRepoStatus(raw: WorkspaceSearchRepoStatusRaw): Worksp workspaceOverlayRoot: raw.workspaceOverlayRoot, phase: raw.phase, snapshotKey: raw.snapshotKey ?? null, + baseHeadCommit: raw.baseHeadCommit ?? null, + workspaceHeadCommit: raw.workspaceHeadCommit ?? null, + baseAdvanceInProgress: raw.baseAdvanceInProgress, + baseAdvanceTargetHead: raw.baseAdvanceTargetHead ?? null, + baseDeltaDepth: raw.baseDeltaDepth, + baseCompactionRecommended: raw.baseCompactionRecommended, lastProbeUnixSecs: raw.lastProbeUnixSecs ?? null, lastRebuildUnixSecs: raw.lastRebuildUnixSecs ?? null, dirtyFiles: raw.dirtyFiles, - rebuildRecommended: raw.rebuildRecommended, activeTaskId: raw.activeTaskId ?? null, probeHealthy: raw.probeHealthy, + workspaceProbePending: raw.workspaceProbePending ?? false, lastError: raw.lastError ?? null, + lastMaintenanceError: raw.lastMaintenanceError ?? null, overlay: raw.overlay ?? null, }; } @@ -156,10 +178,22 @@ function mapWorkspaceSearchTaskStatus( }; } +function mapWorkspaceSearchAutoIndexStatus( + raw: WorkspaceSearchAutoIndexStatusRaw +): NonNullable { + return { + decision: raw.decision, + threshold: raw.threshold, + indexableFiles: raw.indexableFiles ?? null, + reason: raw.reason ?? null, + }; +} + function mapWorkspaceSearchIndexStatus(raw: WorkspaceSearchIndexStatusRaw): WorkspaceSearchIndexStatus { return { repoStatus: mapWorkspaceSearchRepoStatus(raw.repoStatus), activeTask: raw.activeTask ? mapWorkspaceSearchTaskStatus(raw.activeTask) : null, + autoIndex: raw.autoIndex ? mapWorkspaceSearchAutoIndexStatus(raw.autoIndex) : null, }; } diff --git a/src/web-ui/src/infrastructure/api/service-api/tauri-commands.ts b/src/web-ui/src/infrastructure/api/service-api/tauri-commands.ts index 70b83a0892..5dd97d0956 100644 --- a/src/web-ui/src/infrastructure/api/service-api/tauri-commands.ts +++ b/src/web-ui/src/infrastructure/api/service-api/tauri-commands.ts @@ -248,7 +248,13 @@ export type SearchBackendKind = export interface SearchMetadata { backend: SearchBackendKind | string; repoPhase: WorkspaceSearchRepoPhase | string; - rebuildRecommended: boolean; + baseAdvanceInProgress: boolean; + /** + * True when the daemon still owed a worktree reconcile at query time, so these results describe the + * worktree as of the last observation. Interactive search accepts that on purpose rather than + * blocking the panel on a worktree walk. + */ + workspaceProbePending: boolean; candidateDocs: number; matchedLines: number; matchedOccurrences: number; @@ -266,6 +272,8 @@ export type WorkspaceSearchRepoPhase = export type WorkspaceSearchTaskKind = | 'build' | 'rebuild' + | 'advance' + | 'compact' | 'refresh'; export type WorkspaceSearchTaskState = @@ -296,17 +304,35 @@ export interface WorkspaceSearchRepoStatus { workspaceOverlayRoot: string; phase: WorkspaceSearchRepoPhase; snapshotKey?: string | null; + baseHeadCommit?: string | null; + workspaceHeadCommit?: string | null; + /** True while the daemon is advancing the base snapshot toward a newer commit. */ + baseAdvanceInProgress: boolean; + baseAdvanceTargetHead?: string | null; + baseDeltaDepth: number; + baseCompactionRecommended: boolean; lastProbeUnixSecs?: number | null; lastRebuildUnixSecs?: number | null; dirtyFiles: WorkspaceSearchDirtyFiles; - rebuildRecommended: boolean; activeTaskId?: string | null; probeHealthy: boolean; + /** + * True while the daemon still owes a worktree reconcile, which makes `dirtyFiles` and `phase` the + * last observed state rather than the current one. Search results stay usable; they just describe + * a worktree from a moment ago. + */ + workspaceProbePending: boolean; lastError?: string | null; + /** + * Failure of the daemon's last background base-maintenance task. Survives the worktree probes + * that clear `lastError`, so status polling can actually observe it. + */ + lastMaintenanceError?: string | null; overlay?: WorkspaceSearchOverlayStatus | null; } export interface WorkspaceSearchOverlayStatus { + baseManifestId?: string | null; committedSeqNo: number; lastSeqNo: number; uncommittedOps: number; @@ -337,9 +363,33 @@ export interface WorkspaceSearchTaskStatus { error?: string | null; } +/** + * Outcome of BitFun's automatic-index policy. The daemon reports `needs_index` both while the + * policy is still evaluating a workspace and after it deliberately declined to index one, so the + * daemon phase alone cannot explain to the user why nothing is happening. + */ +export type WorkspaceSearchAutoIndexDecision = + | 'pending' + | 'eligible' + | 'belowThreshold' + | 'unsupported'; + +export interface WorkspaceSearchAutoIndexStatus { + decision: WorkspaceSearchAutoIndexDecision; + threshold: number; + /** + * Exact for `belowThreshold`. For `eligible` the count stops as soon as the threshold is + * reached, so it is only a lower bound there. Absent for `pending` and `unsupported`. + */ + indexableFiles?: number | null; + reason?: string | null; +} + export interface WorkspaceSearchIndexStatus { repoStatus: WorkspaceSearchRepoStatus; activeTask?: WorkspaceSearchTaskStatus | null; + /** Absent on transports without a BitFun-side auto-index policy (remote SSH workspaces). */ + autoIndex?: WorkspaceSearchAutoIndexStatus | null; } export interface WorkspaceSearchIndexTaskHandle { diff --git a/src/web-ui/src/locales/en-US/panels/files.json b/src/web-ui/src/locales/en-US/panels/files.json index 92ed6fd520..4f99f04602 100644 --- a/src/web-ui/src/locales/en-US/panels/files.json +++ b/src/web-ui/src/locales/en-US/panels/files.json @@ -23,27 +23,37 @@ "running": "Indexing..." }, "badges": { - "rebuildRecommended": "Rebuild Recommended" + "baseAdvancing": "Index Catching Up", + "probePending": "Last Observed View" }, "phase": { "preparing": "Preparing", "needs_index": "Needs Index", + "no_index_needed": "No index needed", "building": "Building", "ready": "Ready", "tracking_changes": "Tracking Changes", "refreshing": "Refreshing", "limited": "Limited", - "unknown": "Unknown" + "unknown": "Unknown", + "non_git": "Not a Git repo" }, "summary": { "preparing": "Preparing the managed search workspace.", - "needs_index": "Search works with fallback now. Build the index for faster content search.", + "needs_index": "Automatic indexing is evaluating this workspace or has selected fallback search. No manual action is needed.", "building": "Building the workspace index.", "ready": "Managed index is ready. Content search can use the indexed backend.", - "tracking_changes": "Managed index is usable and workspace changes are being tracked. Rebuild is only needed when one is recommended.", + "tracking_changes": "Managed index is usable and workspace changes are being tracked. After a branch switch the index catches up to the new commit automatically.", "refreshing": "Refreshing the workspace index with the latest file changes.", - "limited": "Managed index is limited. Search can still fall back, but rebuild is recommended.", - "unavailable": "Search index status is unavailable right now." + "limited": "Managed index is limited. Search can still fall back; rebuilding the index may restore it.", + "unavailable": "Search index status is unavailable right now.", + "non_git": "This workspace is not a Git repository, so the search index does not apply; content search uses the fallback implementation." + }, + "autoIndex": { + "pending": "Evaluating whether an index is needed.", + "eligible": "Above the indexing threshold; the index task is starting.", + "belowThreshold": "Only {{files}} files, below the {{threshold}} threshold — searching directly is faster.", + "unsupported": "Cannot be indexed: {{reason}}." }, "indicator": { "label": "Workspace index status", @@ -51,13 +61,14 @@ "tones": { "green": "Index ready", "yellow": "Index needs attention", - "gray": "Index not built", - "red": "Index rebuild required" + "gray": "Fallback search available", + "red": "Index unhealthy" }, "progressKnown": "{{processed}} / {{total}}", "progressUnknown": "{{processed}} items processed", "dirtyFiles": "Pending changes: {{modified}} modified, {{deleted}} deleted, {{new}} new", - "rebuildRecommended": "A rebuild is recommended.", + "baseAdvancing": "Advancing the index to the latest commit.", + "probePending": "Reconciling worktree changes; the dirty file counts above are from the last observation.", "probeDegraded": "Workspace probe is degraded and the indexed workspace view may stop tracking changes.", "hoverTooltip": "Flashgrep index: {{status}}" }, diff --git a/src/web-ui/src/locales/zh-CN/panels/files.json b/src/web-ui/src/locales/zh-CN/panels/files.json index ad772cd18a..469f0f47d8 100644 --- a/src/web-ui/src/locales/zh-CN/panels/files.json +++ b/src/web-ui/src/locales/zh-CN/panels/files.json @@ -23,27 +23,37 @@ "running": "索引中..." }, "badges": { - "rebuildRecommended": "建议重建" + "baseAdvancing": "索引追赶中", + "probePending": "上次观测视图" }, "phase": { "preparing": "准备中", "needs_index": "未建索引", + "no_index_needed": "无需索引", "building": "建立中", "ready": "索引可用", "tracking_changes": "跟踪变更中", "refreshing": "刷新中", "limited": "受限", - "unknown": "未知" + "unknown": "未知", + "non_git": "非 Git 仓库" }, "summary": { "preparing": "正在准备托管搜索工作区。", - "needs_index": "当前搜索仍可通过 fallback 使用,建立索引后内容搜索会更快。", + "needs_index": "自动索引策略正在评估当前工作区,或已选择继续使用回退搜索,无需手动操作。", "building": "正在为当前工作区建立索引。", "ready": "索引已就绪,内容搜索可优先走索引后端。", - "tracking_changes": "索引当前可用,工作区变更正在被跟踪。只有出现建议重建时,才需要关注重建。", + "tracking_changes": "索引当前可用,工作区变更正在被跟踪。切换分支后索引会自动追赶到最新提交。", "refreshing": "正在根据最新文件变更刷新索引。", - "limited": "索引能力受限。搜索仍可 fallback,但建议重建。", - "unavailable": "当前无法获取索引状态。" + "limited": "索引能力受限。搜索仍可 fallback,可尝试重建索引恢复。", + "unavailable": "当前无法获取索引状态。", + "non_git": "当前工作区不是 Git 仓库,索引搜索不适用;内容搜索会走回退实现。" + }, + "autoIndex": { + "pending": "正在评估是否需要建索引。", + "eligible": "已达建索引门槛,索引任务即将开始。", + "belowThreshold": "工作区仅 {{files}} 个文件,未达 {{threshold}} 门槛,直接搜索更快。", + "unsupported": "无法建立索引:{{reason}}。" }, "indicator": { "label": "工作区索引状态", @@ -51,13 +61,14 @@ "tones": { "green": "索引正常", "yellow": "索引需关注", - "gray": "尚未建索引", - "red": "需要重建索引" + "gray": "回退搜索可用", + "red": "索引异常" }, "progressKnown": "{{processed}} / {{total}}", "progressUnknown": "已处理 {{processed}} 项", "dirtyFiles": "待同步变更:修改 {{modified}},删除 {{deleted}},新增 {{new}}", - "rebuildRecommended": "当前建议执行重建。", + "baseAdvancing": "正在把索引推进到最新提交。", + "probePending": "正在核对工作区改动,当前脏文件数是上次观测结果。", "probeDegraded": "工作区探测状态异常,工作区索引视图可能不再持续跟进变更。", "hoverTooltip": "Flashgrep 索引:{{status}}" }, diff --git a/src/web-ui/src/locales/zh-TW/panels/files.json b/src/web-ui/src/locales/zh-TW/panels/files.json index 338d68bf0d..445a7e1507 100644 --- a/src/web-ui/src/locales/zh-TW/panels/files.json +++ b/src/web-ui/src/locales/zh-TW/panels/files.json @@ -23,27 +23,37 @@ "running": "索引中..." }, "badges": { - "rebuildRecommended": "建議重建" + "baseAdvancing": "索引追趕中", + "probePending": "上次觀測檢視" }, "phase": { "preparing": "準備中", "needs_index": "未建索引", + "no_index_needed": "無需索引", "building": "建立中", "ready": "索引可用", "tracking_changes": "跟蹤變更中", "refreshing": "重新整理中", "limited": "受限", - "unknown": "未知" + "unknown": "未知", + "non_git": "非 Git 倉庫" }, "summary": { "preparing": "正在準備托管搜尋工作區。", - "needs_index": "目前搜尋仍可透過 fallback 使用,建立索引後內容搜尋會更快。", + "needs_index": "自動索引策略正在評估目前工作區,或已選擇繼續使用回退搜尋,無需手動操作。", "building": "正在為目前工作區建立索引。", "ready": "索引已就緒,內容搜尋可優先走索引後端。", - "tracking_changes": "索引目前可用,工作區變更正在被跟蹤。只有出現建議重建時,才需要關注重建。", + "tracking_changes": "索引目前可用,工作區變更正在被跟蹤。切換分支後索引會自動追趕到最新提交。", "refreshing": "正在根據最新檔案變更重新整理索引。", - "limited": "索引能力受限。搜尋仍可 fallback,但建議重建。", - "unavailable": "目前無法獲取索引狀態。" + "limited": "索引能力受限。搜尋仍可 fallback,可嘗試重建索引恢復。", + "unavailable": "目前無法獲取索引狀態。", + "non_git": "目前工作區不是 Git 倉庫,索引搜尋不適用;內容搜尋會走回退實作。" + }, + "autoIndex": { + "pending": "正在評估是否需要建索引。", + "eligible": "已達建索引門檻,索引任務即將開始。", + "belowThreshold": "工作區僅 {{files}} 個檔案,未達 {{threshold}} 門檻,直接搜尋更快。", + "unsupported": "無法建立索引:{{reason}}。" }, "indicator": { "label": "工作區索引狀態", @@ -51,13 +61,14 @@ "tones": { "green": "索引正常", "yellow": "索引需關注", - "gray": "尚未建索引", - "red": "需要重建索引" + "gray": "回退搜尋可用", + "red": "索引異常" }, "progressKnown": "{{processed}} / {{total}}", "progressUnknown": "已處理 {{processed}} 項", "dirtyFiles": "待同步變更:修改 {{modified}},刪除 {{deleted}},新增 {{new}}", - "rebuildRecommended": "目前建議執行重建。", + "baseAdvancing": "正在把索引推進到最新提交。", + "probePending": "正在核對工作區改動,當前臟檔案數是上次觀測結果。", "probeDegraded": "工作區探測狀態異常,工作區索引視圖可能不再持續跟進變更。", "hoverTooltip": "Flashgrep 索引:{{status}}" }, diff --git a/src/web-ui/src/tools/file-explorer/search/useWorkspaceSearchIndex.ts b/src/web-ui/src/tools/file-explorer/search/useWorkspaceSearchIndex.ts index 52946f4e98..ea9fc8cb62 100644 --- a/src/web-ui/src/tools/file-explorer/search/useWorkspaceSearchIndex.ts +++ b/src/web-ui/src/tools/file-explorer/search/useWorkspaceSearchIndex.ts @@ -15,10 +15,27 @@ const log = createLogger('useWorkspaceSearchIndex'); const ACTIVE_TASK_POLL_MS = 1000; const IDLE_STATUS_POLL_MS = 5000; -function isWorkspaceSearchUnavailableError(message: string): boolean { - return message.includes('Workspace search is disabled') +// Kept in sync with `NON_GIT_WORKSPACE_MESSAGE` in `src/apps/desktop/src/api/search_api.rs`. +const NON_GIT_WORKSPACE_MESSAGE = 'Workspace search requires a Git worktree with a HEAD commit'; + +export type WorkspaceSearchUnsupportedReason = 'non_git' | 'other'; + +// A workspace that flashgrep cannot index is not an error state: content search silently falls +// back, so the caller should stop polling and render a neutral indicator rather than a red one. +function workspaceSearchUnsupportedReason( + message: string +): WorkspaceSearchUnsupportedReason | null { + if (message.includes(NON_GIT_WORKSPACE_MESSAGE)) { + return 'non_git'; + } + if ( + message.includes('Workspace search is disabled') || message.includes('Workspace search daemon is unavailable') - || message.includes('Remote workspace search status is not managed'); + || message.includes('Remote workspace search status is not managed') + ) { + return 'other'; + } + return null; } export interface UseWorkspaceSearchIndexOptions { @@ -34,6 +51,7 @@ export interface UseWorkspaceSearchIndexResult { error: string | null; supported: boolean; hasActiveTask: boolean; + unsupportedReason: WorkspaceSearchUnsupportedReason | null; refreshStatus: (silent?: boolean) => Promise; buildIndex: () => Promise; rebuildIndex: () => Promise; @@ -54,8 +72,9 @@ export function useWorkspaceSearchIndex( const [refreshing, setRefreshing] = useState(false); const [actionRunning, setActionRunning] = useState(false); const [error, setError] = useState(null); - const [backendSupported, setBackendSupported] = useState(true); - const supported = Boolean(workspacePath && enabled && backendSupported); + const [unsupportedReason, setUnsupportedReason] = + useState(null); + const supported = Boolean(workspacePath && enabled && unsupportedReason === null); const mountedRef = useRef(true); const pollTimerRef = useRef | null>(null); @@ -98,8 +117,9 @@ export function useWorkspaceSearchIndex( return null; } const message = err instanceof Error ? err.message : 'Failed to load search index status'; - if (isWorkspaceSearchUnavailableError(message)) { - setBackendSupported(false); + const reason = workspaceSearchUnsupportedReason(message); + if (reason) { + setUnsupportedReason(reason); setIndexStatus(null); setError(null); return null; @@ -135,18 +155,22 @@ export function useWorkspaceSearchIndex( ? await workspaceAPI.buildSearchIndex(workspacePath) : await workspaceAPI.rebuildSearchIndex(workspacePath); if (mountedRef.current) { - setIndexStatus({ + // The task handle carries no auto-index decision, but a manual build does not change + // one either, so the last known decision is kept until the next status refresh. + setIndexStatus((previous) => ({ repoStatus: result.repoStatus, activeTask: result.task, - }); + autoIndex: previous?.autoIndex ?? null, + })); setError(null); } return result; } catch (err) { if (mountedRef.current) { const message = err instanceof Error ? err.message : `Failed to ${action} search index`; - if (isWorkspaceSearchUnavailableError(message)) { - setBackendSupported(false); + const reason = workspaceSearchUnsupportedReason(message); + if (reason) { + setUnsupportedReason(reason); setIndexStatus(null); setError(null); } else { @@ -175,7 +199,7 @@ export function useWorkspaceSearchIndex( }, [clearPollTimer]); useEffect(() => { - setBackendSupported(true); + setUnsupportedReason(null); }, [enabled, workspacePath]); useEffect(() => { @@ -226,6 +250,7 @@ export function useWorkspaceSearchIndex( actionRunning, error, supported, + unsupportedReason, hasActiveTask: isTaskActive(indexStatus), refreshStatus, buildIndex, @@ -241,6 +266,7 @@ export function useWorkspaceSearchIndex( refreshStatus, refreshing, supported, + unsupportedReason, ] ); }