feat(vector-index): OpenSearch copy-on-dump into immutable generation indices (#343)

docs/source/concepts.rst

@@ -124,3 +124,21 @@ AutoIntent's design emphasizes modularity and extensibility:
    Framework can be extended with custom embedding models, scoring algorithms, and decision strategies while maintaining compatibility with the AutoML optimization pipeline.
 
 This modular design ensures that AutoIntent can evolve with advances in NLP research while maintaining stability and backward compatibility for existing users.
+
+Vector Index Lifecycle
+======================
+
+All vector-index backends follow one contract: the **live index a module fits into is transient training scratch**, owned by whichever instance is currently fitting (each ``fit()`` replaces its contents); **durability comes from** ``dump()`` **artifacts**. During hyperparameter search only the best module of each node is dumped, and trials keep rewriting the scratch index after the best one — so never serve from the live index; serve from a loaded dump.
+
+How ``dump()`` achieves durability differs by backend:
+
+- **Faiss** writes a self-contained directory (documents + binary index).
+- **OpenSearch** performs *copy-on-dump*: the live index is copied server-side (``_reindex``) into a write-blocked *generation* index named ``{base}-best-{id}``, and the dump directory records it in ``remote_manifest.json``. The corpus never leaves the cluster. ``Pipeline.load`` binds to that immutable generation, verifies its identity, and never writes; it fails loudly if the generation was deleted or recreated.
+
+Notes specific to the OpenSearch backend:
+
+- **Cleanup.** Delete dumps with :func:`autointent.remove_module_dump` — it removes the referenced generation index from the cluster along with the directory. A plain ``rm -rf`` strands the generation (recoverable: generations are pattern-named ``*-best-*``, so ops can also enforce an ISM age policy). AutoIntent's optimizer uses this helper automatically when a new best trial replaces the previous one, so during optimization the steady-state cluster footprint is the live scratch index plus one generation per node type; a later ``Pipeline.dump()`` re-dump of an already-optimized pipeline adds one more generation per node, which lingers until the corresponding dump directory is deleted.
+- **Naming.** Avoid naming your own live indices with a ``-best-`` infix — ``{base}-best-*`` is the namespace AutoIntent uses for generations and the serving alias.
+- **Portability.** A dumped OpenSearch pipeline is portable only as far as the cluster: copying the dump directory to an environment that cannot reach the same cluster carries a dangling reference. This is inherent to keeping the corpus in the engine.
+- **Querying an existing collection.** Do not point ``OpenSearchConfig.index_name`` at a collection you want to keep — ``fit()`` clears it (fit-replaces). To serve an existing corpus read-only, fit and dump once, then use ``Pipeline.load`` + ``predict``: loaded pipelines only ever read their immutable generation.
+- **Parallel trials.** Hyperparameter search with ``n_jobs > 1`` would share one live scratch index across concurrently fitting trials; keep ``n_jobs = 1`` when using the OpenSearch backend.

src/autointent/__init__.py

@@ -1,7 +1,7 @@
 """This is AutoIntent API reference."""
 
 from ._logging import setup_logging
-from ._wrappers import Ranker, Embedder, VectorIndex
+from ._wrappers import Ranker, Embedder, VectorIndex, remove_module_dump
 from ._dataset import Dataset
 from ._hash import Hasher
 from .context import Context, load_dataset
@@ -19,5 +19,6 @@
     "Ranker",
     "VectorIndex",
     "load_dataset",
+    "remove_module_dump",
     "setup_logging",
 ]

src/autointent/_utils.py

@@ -25,5 +25,3 @@ def detect_device() -> str:
     if torch.mps.is_available():
         return "mps"
     return "cpu"
-
-

src/autointent/_wrappers/__init__.py

@@ -1,7 +1,7 @@
 from .ranker import Ranker
 from .embedder import Embedder
-from .vector_index import VectorIndex
+from .vector_index import VectorIndex, remove_module_dump
 from .base_torch_module import BaseTorchModuleWithVocab
 from .base_torch_module import BaseTorchModule
 
-__all__ = ["BaseTorchModule", "BaseTorchModuleWithVocab", "Embedder", "Ranker", "VectorIndex"]
+__all__ = ["BaseTorchModule", "BaseTorchModuleWithVocab", "Embedder", "Ranker", "VectorIndex", "remove_module_dump"]

src/autointent/_wrappers/vector_index/__init__.py

@@ -1,3 +1,4 @@
+from .remote_dumps import remove_module_dump
 from .vector_index import VectorIndex
 
-__all__ = ["VectorIndex"]
+__all__ = ["VectorIndex", "remove_module_dump"]

src/autointent/_wrappers/vector_index/base_backend.py

@@ -13,6 +13,15 @@
     from autointent.custom_types import Document
 
 
+MANIFEST_FILENAME = "remote_manifest.json"
+"""Marker file inside a dump directory: the dump references remote cluster state.
+
+Any backend whose ``dump()`` leaves data in an external engine writes this file
+(``{"engine": ..., "index": ..., "dump_id": ...}``) so that dump-deletion tooling
+can clean up the referenced cluster index (see ``remote_dumps.remove_module_dump``).
+"""
+
+
 class BaseIndexBackend(ABC):
     @abstractmethod
     def __init__(self, config: VectorIndexConfig, vector_size: int) -> None: ...