Valid values: {@code DEBUG}, {@code INFO}, {@code WARN}, {@code ERROR} (default: {@code WARN}). + * Set to {@code ERROR} or {@code DEBUG} to increase/decrease visibility during migration QA.
+ */ + String SHADOW_WRITE_LOG_LEVEL_KEY = "DOTCMS_SHADOW_WRITE_LOG_LEVEL"; + + /** + * Logs an OS shadow write failure at the level configured by + * {@value #SHADOW_WRITE_LOG_LEVEL_KEY} (default: {@code WARN}). + * + * @param clazz the class to attribute the log entry to + * @param message the log message + * @param t the throwable, or {@code null} if none + */ + static void logShadowWriteFailure(final Class clazz, + final String message, + final Throwable t) { + final String level = Config.getStringProperty(SHADOW_WRITE_LOG_LEVEL_KEY, "WARN") + .toUpperCase(); + switch (level) { + case "DEBUG": Logger.debug(clazz, message, t); break; + case "INFO": Logger.info(clazz, message); break; + case "ERROR": Logger.error(clazz, message, t); break; + default: Logger.warn(clazz, message, t); break; + } + } + // ------------------------------------------------------------------------- // Migration phase // ------------------------------------------------------------------------- diff --git a/dotCMS/src/main/java/com/dotcms/content/index/PhaseRouter.java b/dotCMS/src/main/java/com/dotcms/content/index/PhaseRouter.java index 905dc4c1f6e3..25298ed286d6 100644 --- a/dotCMS/src/main/java/com/dotcms/content/index/PhaseRouter.java +++ b/dotCMS/src/main/java/com/dotcms/content/index/PhaseRouter.java @@ -4,6 +4,8 @@ import static com.dotcms.content.index.IndexConfigHelper.isMigrationNotStarted; import static com.dotcms.content.index.IndexConfigHelper.isReadEnabled; +import static com.dotcms.content.index.IndexConfigHelper.logShadowWriteFailure; + import com.dotmarketing.util.Logger; import java.util.List; import java.util.function.Consumer; @@ -246,7 +248,7 @@ public boolean writeBoolean(final FunctionThese tests document the current, observable behavior of each API method + * across migration phases without requiring a running Elasticsearch or OpenSearch cluster. + * All infrastructure is replaced by in-memory fakes injected via the package-private + * testing constructor.
+ */ +public class ContentletIndexAPIImplPhaseTest { + + // ── Logical index names ─────────────────────────────────────────────────── + /** ES index timestamp suffix (created first, in Phase 0). */ + private static final String ES_WORKING = "working_T0"; + private static final String ES_LIVE = "live_T0"; + + /** OS index timestamp suffix (created during migration catchup). */ + private static final String OS_WORKING = "working_T1"; + private static final String OS_LIVE = "live_T1"; + + /** Cluster prefix prepended by both providers' {@code toPhysicalName()}. */ + private static final String CLUSTER_PREFIX = "cluster_test."; + + // ── Test teardown ───────────────────────────────────────────────────────── + + @After + public void clearPhase() { + Config.setProperty(FLAG_KEY, null); + } + + // ========================================================================= + // isDotCMSIndexName — purely syntactic (prefix check, no provider query) + // ========================================================================= + + /** + * Given Scenario: any migration phase. + * The method is purely syntactic: it checks whether the name starts with + * {@code "working_"} or {@code "live_"}. + * + * When : isDotCMSIndexName() is called with the ES logical name. + * Then : returns true — the ES name is a valid dotCMS index name. + */ + @Test + public void test_isDotCMSIndexName_esLogicalName_isTrue() { + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of(ES_WORKING, ES_LIVE)), + new FakeIndiciesAPI(), + new FakeVersionedIndicesAPI()); + + assertTrue("ES working name must be recognised as a dotCMS index", + api.isDotCMSIndexName(ES_WORKING)); + assertTrue("ES live name must be recognised as a dotCMS index", + api.isDotCMSIndexName(ES_LIVE)); + } + + /** + * Given Scenario: any migration phase. + * When : isDotCMSIndexName() is called with the OS logical name. + * Then : returns true — even though only OS has this index, the name itself + * starts with "working_" so it passes the syntactic check. + * The method does NOT query which provider actually holds the index. + */ + @Test + public void test_isDotCMSIndexName_osLogicalName_isTrue() { + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + new FakeIndiciesAPI(), + new FakeVersionedIndicesAPI()); + + assertTrue("OS working name must be recognised as a dotCMS index (syntactic check only)", + api.isDotCMSIndexName(OS_WORKING)); + assertTrue("OS live name must be recognised as a dotCMS index", + api.isDotCMSIndexName(OS_LIVE)); + } + + /** + * Given Scenario: any phase. + * When : isDotCMSIndexName() is called with a physical name (cluster prefix included). + * Then : returns false — physical names do NOT start with "working_" or "live_". + * Callers must strip the cluster prefix before invoking this method. + */ + @Test + public void test_isDotCMSIndexName_physicalNameWithClusterPrefix_isFalse() { + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + new FakeIndiciesAPI(), + new FakeVersionedIndicesAPI()); + + assertFalse("Physical name with cluster prefix must NOT be recognised", + api.isDotCMSIndexName(CLUSTER_PREFIX + ES_WORKING)); + assertFalse("Physical OS name with cluster prefix must NOT be recognised", + api.isDotCMSIndexName(CLUSTER_PREFIX + OS_WORKING)); + } + + /** + * Given Scenario: any phase. + * When : isDotCMSIndexName() is called with a name that has no recognised dotCMS prefix. + * Then : returns false. + */ + @Test + public void test_isDotCMSIndexName_unknownPrefix_isFalse() { + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + new FakeIndiciesAPI(), + new FakeVersionedIndicesAPI()); + + assertFalse("Unrecognised prefix must return false", + api.isDotCMSIndexName("notadotcmsindex_T0")); + assertFalse("Null must return false", api.isDotCMSIndexName(null)); + } + + // ========================================================================= + // listDotCMSIndices — pure delegation to IndexAPI.getIndices(true, false) + // ========================================================================= + + /** + * Given Scenario: Phase 0 (ES only). + * The fake IndexAPI is pre-loaded with ES indices only. + * When : listDotCMSIndices() is called. + * Then : returns exactly the ES indices — OS is not consulted in Phase 0. + * The phase-awareness is encapsulated inside IndexAPIImpl (the real + * implementation); ContentletIndexAPIImpl simply delegates. + */ + @Test + public void test_listDotCMSIndices_phase0_esIndicesOnly() { + setPhase(0); + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of(ES_WORKING, ES_LIVE)), + new FakeIndiciesAPI(), + new FakeVersionedIndicesAPI()); + + final ListObservation: in Phase 0 the ES store can be pointed at an index + * name that does not physically exist in the ES cluster, because activateIndex is + * a pure pointer-store update with no existence check.
+ */ + @Test + public void test_activateIndex_phase0_osName_goesToEsStore_noOsWrite() throws DotDataException { + setPhase(0); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of(ES_WORKING, ES_LIVE)), + fakeIndicie, + fakeVersioned); + + api.activateIndex(OS_WORKING); // OS name passed in Phase 0 + + assertEquals("ES store must record the OS physical name even in Phase 0", + CLUSTER_PREFIX + OS_WORKING, fakeIndicie.loadIndicies().getWorking()); + assertNull("OS store must not be touched in Phase 0", fakeVersioned.stored); + } + + /** + * Given Scenario: Phase 2 (dual-write, OS reads). Caller uses the ES index name T0. + * When : activateIndex("working_T0") is called. + * Then : ES store is updated with T0 (physical). + * OS store mirror is also updated — but it receives the SAME logical name T0, + * even though OS physically has T1. The mirror writes the name as-is, without + * checking which index actually exists in the OS cluster. + * + *Key observation: after this call, the OS pointer store records + * "working_T0" as the active working index — a mismatch with the physical OS index. + * This is expected behavior during the catch-up window.
+ */ + @Test + public void test_activateIndex_phase2_esName_mirroredToOsStore() throws DotDataException { + setPhase(2); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + fakeIndicie, + fakeVersioned); + + api.activateIndex(ES_WORKING); + + // ES pointer store is updated ✓ + assertEquals("ES store must record the physical working name", + CLUSTER_PREFIX + ES_WORKING, fakeIndicie.loadIndicies().getWorking()); + + // OS store receives a mirror of the same logical name + assertTrue("OS store must be populated in Phase 2", + fakeVersioned.stored != null); + assertEquals("OS store must contain the mirrored working name (even if OS has a different physical index)", + CLUSTER_PREFIX + ES_WORKING, + fakeVersioned.stored.working().orElse(null)); + } + + /** + * Given Scenario: Phase 2 (dual-write, OS reads). Caller uses the OS index name T1. + * When : activateIndex("working_T1") is called. + * Then : ES store is updated with T1 (even though ES physically has T0). + * OS store is also updated with T1 — in this case both stores are correct + * because the caller used the OS-native name. + */ + @Test + public void test_activateIndex_phase2_osName_updatesBothStores() throws DotDataException { + setPhase(2); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + fakeIndicie, + fakeVersioned); + + api.activateIndex(OS_WORKING); + + assertEquals("ES store must record the OS physical name", + CLUSTER_PREFIX + OS_WORKING, fakeIndicie.loadIndicies().getWorking()); + assertEquals("OS store must record its own physical name", + CLUSTER_PREFIX + OS_WORKING, + fakeVersioned.stored.working().orElse(null)); + } + + /** + * Given Scenario: Phase 3 (OS only). + * When : activateIndex("working_T1") is called. + * Then : only the OS store (versionedIndicesAPI) is updated. + * ES store (legacyIndiciesAPI) is NOT touched — ES is decommissioned. + */ + @Test + public void test_activateIndex_phase3_onlyOsStoreUpdated() throws DotDataException { + setPhase(3); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), + fakeIndicie, + fakeVersioned); + + api.activateIndex(OS_WORKING); + + assertNull("ES store must NOT be touched in Phase 3 (ES is decommissioned)", + fakeIndicie.loadIndicies().getWorking()); + assertEquals("OS store must record the working name", + CLUSTER_PREFIX + OS_WORKING, + fakeVersioned.stored.working().orElse(null)); + } + + // ========================================================================= + // deactivateIndex — clears a slot from the pointer stores by index type + // ========================================================================= + + /** + * Given Scenario: Phase 0 (ES only). ES store has working=T0, live=T0-live. + * When : deactivateIndex("working_T0") is called. + * Then : ES store working slot is cleared (null); live slot is preserved. + * OS store is NOT touched. + */ + @Test + public void test_deactivateIndex_phase0_clearsEsWorkingSlotOnly() + throws DotDataException, IOException { + setPhase(0); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + fakeIndicie.setWorking(CLUSTER_PREFIX + ES_WORKING); + fakeIndicie.setLive(CLUSTER_PREFIX + ES_LIVE); + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), fakeIndicie, fakeVersioned); + + api.deactivateIndex(ES_WORKING); + + assertNull("Working slot must be cleared after deactivation", + fakeIndicie.loadIndicies().getWorking()); + assertEquals("Live slot must be preserved", + CLUSTER_PREFIX + ES_LIVE, fakeIndicie.loadIndicies().getLive()); + assertNull("OS store must not be touched in Phase 0", fakeVersioned.stored); + } + + /** + * Given Scenario: Phase 2 (dual-write). Both ES and OS stores have their respective + * working indices recorded (T0 in ES, T1 in OS). + * When : deactivateIndex("working_T0") is called. + * Then : ES store working slot is cleared. + * OS store working slot is ALSO cleared — the deactivation clears the slot + * by INDEX TYPE (WORKING), not by the specific index name (T0 vs T1). + * The OS live slot is preserved. + * + *Key observation: deactivateIndex identifies which slot to clear + * via the index-type prefix ("working_" / "live_"), not via the exact name. + * Passing "working_T0" in Phase 2 will clear the OS working slot even though OS + * physically records "working_T1" there.
+ */ + @Test + public void test_deactivateIndex_phase2_clearsBothStoresByIndexType() + throws DotDataException, IOException { + setPhase(2); + final FakeIndiciesAPI fakeIndicie = new FakeIndiciesAPI(); + fakeIndicie.setWorking(CLUSTER_PREFIX + ES_WORKING); + fakeIndicie.setLive(CLUSTER_PREFIX + ES_LIVE); + + final FakeVersionedIndicesAPI fakeVersioned = new FakeVersionedIndicesAPI(); + // Pre-populate OS store: OS working = T1, OS live = T1-live + fakeVersioned.stored = VersionedIndicesImpl.builder() + .working(CLUSTER_PREFIX + OS_WORKING) + .live(CLUSTER_PREFIX + OS_LIVE) + .build(); + + final ContentletIndexAPIImpl api = buildApi( + new FakeIndexAPI(List.of()), fakeIndicie, fakeVersioned); + + // Deactivate using the ES index name — but the SLOT (WORKING) is cleared in both stores + api.deactivateIndex(ES_WORKING); + + assertNull("ES working slot must be cleared", + fakeIndicie.loadIndicies().getWorking()); + assertEquals("ES live slot must be preserved", + CLUSTER_PREFIX + ES_LIVE, fakeIndicie.loadIndicies().getLive()); + + // OS mirror: working slot cleared (by index type), live preserved + assertNull("OS working slot must also be cleared (deactivation is by index type, not name)", + fakeVersioned.stored.working().orElse(null)); + assertEquals("OS live slot must be preserved", + CLUSTER_PREFIX + OS_LIVE, fakeVersioned.stored.live().orElse(null)); + } + + // ========================================================================= + // Factory and helpers + // ========================================================================= + + private static ContentletIndexAPIImpl buildApi( + final FakeIndexAPI fakeIndex, + final FakeIndiciesAPI fakeIndicie, + final FakeVersionedIndicesAPI fakeVersioned) { + return new ContentletIndexAPIImpl( + new FakeContentletIndexOperations(), + new FakeContentletIndexOperations(), + fakeIndex, + fakeIndicie, + fakeVersioned); + } + + private static void setPhase(final int ordinal) { + Config.setProperty(FLAG_KEY, String.valueOf(ordinal)); + } + + // ========================================================================= + // Fake implementations — in-memory stubs with no vendor dependencies + // ========================================================================= + + /** + * In-memory {@link IndexAPI} stub. + * Only the three methods used by the target methods are implemented; + * all others throw {@link UnsupportedOperationException}. + */ + static class FakeIndexAPI implements IndexAPI { + + private final ListOnly {@link #toPhysicalName} is implemented (the default interface method is + * overridden to avoid calling a real {@link IndexAPI}). All bulk and lifecycle + * operations throw {@link UnsupportedOperationException} since none of the + * tested methods invoke them.
+ */ + static class FakeContentletIndexOperations implements ContentletIndexOperations { + + @Override + public String toPhysicalName(final String indexName) { + return indexName.startsWith(CLUSTER_PREFIX) ? indexName : CLUSTER_PREFIX + indexName; + } + + @Override + public IndexAPI indexAPI() { + throw new UnsupportedOperationException("indexAPI() not used by tests"); + } + + // ── unneeded methods ───────────────────────────────────────────────── + + @Override public IndexBulkRequest createBulkRequest() { throw new UnsupportedOperationException(); } + @Override public void addIndexOp(IndexBulkRequest r, String i, String d, String j) { throw new UnsupportedOperationException(); } + @Override public void addDeleteOp(IndexBulkRequest r, String i, String d) { throw new UnsupportedOperationException(); } + @Override public void setRefreshPolicy(IndexBulkRequest r, IndexBulkRequest.RefreshPolicy p) { throw new UnsupportedOperationException(); } + @Override public void putToIndex(IndexBulkRequest r) { throw new UnsupportedOperationException(); } + @Override public IndexBulkProcessor createBulkProcessor(IndexBulkListener l) { throw new UnsupportedOperationException(); } + @Override public void addIndexOpToProcessor(IndexBulkProcessor p, String i, String d, String j) { throw new UnsupportedOperationException(); } + @Override public void addDeleteOpToProcessor(IndexBulkProcessor p, String i, String d) { throw new UnsupportedOperationException(); } + @Override public boolean createContentIndex(String n, int s) { throw new UnsupportedOperationException(); } + @Override public void removeContentFromIndexByContentType(ContentType t) { throw new UnsupportedOperationException(); } + @Override public long getIndexDocumentCount(String n) { throw new UnsupportedOperationException(); } + } +} diff --git a/dotCMS/src/test/java/com/dotcms/content/index/PhaseRouterTest.java b/dotCMS/src/test/java/com/dotcms/content/index/PhaseRouterTest.java new file mode 100644 index 000000000000..c4323808cefe --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/content/index/PhaseRouterTest.java @@ -0,0 +1,340 @@ +package com.dotcms.content.index; + +import static com.dotcms.content.index.IndexConfigHelper.MigrationPhase.FLAG_KEY; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.dotmarketing.util.Config; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import org.junit.After; +import org.junit.Test; + +/** + * Unit tests for {@link PhaseRouter} fire-and-forget and failure-propagation behaviour. + * + *In a common dual-write catch-up scenario the ES and OS indices carry different + * timestamp suffixes. For example, ES has {@code working_T0} and OS has {@code working_T1}. + * Calling an index lifecycle operation ({@code closeIndex}, {@code openIndex}, {@code delete}) + * with the ES index name succeeds on ES but throws + * {@code [index_not_found_exception]} on OS, because OS does not know about {@code working_T0}.
+ * + *The routing table below shows who is primary (failure propagates) and who is + * shadow (failure is fire-and-forget) in each phase:
+ * + *
+ * Phase │ Primary │ Shadow │ ES index = T0, OS index = T1 → closeIndex("T0")
+ * ──────┼─────────┼────────┼────────────────────────────────────────────────────
+ * 0 │ ES only │ — │ ES ok → success
+ * 1 │ ES │ OS │ ES ok, OS throws (T0 not in OS) → success (fire-and-forget)
+ * 2 │ OS │ ES │ OS throws (T0 not in OS, primary) → THROWS
+ * 3 │ OS only │ — │ OS throws → THROWS
+ *
+ *
+ * The inverse scenario (OS has T1, ES has T0, caller uses T1) is also tested for Phase 2 + * to confirm that ES shadow failures are fire-and-forget when OS is the primary reader.
+ */ +public class PhaseRouterTest { + + // ── Test setup / teardown ───────────────────────────────────────────────── + + @After + public void clearPhase() { + Config.setProperty(FLAG_KEY, null); + } + + // ========================================================================= + // write() — void unchecked fan-out + // Used by: IndexAPIImpl.closeIndex(), openIndex(), createAlias() + // ========================================================================= + + /** + * Given Scenario: Phase 1 (dual-write, ES reads). ES has "working_T0"; OS has "working_T1". + * When : closeIndex("working_T0") is fanned out to both providers. + * Then : ES (primary) succeeds; OS (shadow) throws index_not_found — exception is swallowed. + * Both providers were called (shadow is never skipped even on primary success). + */ + @Test + public void test_write_phase1_osShadowIndexNotFound_isSwallowed() { + final AtomicBoolean esCalled = new AtomicBoolean(); + final AtomicBoolean osCalled = new AtomicBoolean(); + + // ES has working_T0 → operation succeeds + final Runnable esAction = () -> esCalled.set(true); + + // OS has working_T1 → operation fails (index_not_found for T0) + final Runnable osAction = () -> { + osCalled.set(true); + throw new RuntimeException( + "[index_not_found_exception] no such index [working_T0] — OS index is working_T1"); + }; + + final PhaseRouterThe tests simulate the real catch-up situation that arises during the ES→OS migration: + * Elasticsearch (or an ES-compatible cluster) was the original search backend and holds index + * {@code working_T0} / {@code live_T0}. OpenSearch was brought up later and received index + * {@code working_T1} / {@code live_T1}. Both clusters are live simultaneously; the migration + * phase flag controls which provider is treated as primary.
+ * + *Some tests require separate ES and OS clusters to observe cluster-isolation + * behavior (e.g. "Phase 0 shows only ES indices"). When both clients point to the same cluster + * — as happens in the {@code opensearch-upgrade} Maven profile where + * {@code DOT_ES_ENDPOINTS == OS_ENDPOINTS} — those tests are automatically skipped via + * {@link org.junit.Assume}. Tests that only verify routing logic or DB pointer state work + * correctly in both single- and two-cluster setups.
+ * + *+ * ./mvnw verify -pl :dotcms-integration \ + * -Dcoreit.test.skip=false \ + * -Dopensearch.upgrade.test=true \ + * -Dit.test=ContentletIndexAPIImplMigrationIT + *+ * + * @author Fabrizzio Araya + */ +@ApplicationScoped +@RunWith(DataProviderWeldRunner.class) +public class ContentletIndexAPIImplMigrationIT extends IntegrationTestBase { + + // ── Unique suffix prevents cross-run index name collisions ──────────────── + private static final String RUN_ID = + UUID.randomUUID().toString().replace("-", "").substring(0, 8); + + /** + * ES index name — represents the index created during Phase 0 (before OS existed). + */ + private static final String ES_WORKING = "working_t0_" + RUN_ID; + private static final String ES_LIVE = "live_t0_" + RUN_ID; + + /** + * OS index name — represents the index created during migration catch-up. + * Different timestamp suffix → different from the ES name. + */ + private static final String OS_WORKING = "working_t1_" + RUN_ID; + private static final String OS_LIVE = "live_t1_" + RUN_ID; + + /** + * Name used for the dual-write fan-out test — a single logical name that + * {@code createContentIndex()} sends to both providers simultaneously. + */ + private static final String DUAL_WORKING = "working_dual_" + RUN_ID; + + /** + * A name that both ES and OS reject at the cluster level: spaces are not allowed in index names. + * Used to verify that cluster-level validation errors always propagate — they are NOT + * swallowed by the fire-and-forget mechanism (which only applies to shadow {@code index_not_found}). + */ + private static final String INVALID_INDEX_NAME = "invalid name with spaces " + RUN_ID; + + /** + * A name that matches the {@code working} prefix (so {@link IndexType#WORKING} recognises it) + * but was never created in any cluster. + * Used to verify that {@code deactivateIndex} is a DB-pointer-only operation: it clears + * the slot based on the name pattern, without checking cluster existence. + */ + private static final String GHOST_WORKING = "working_ghost_" + RUN_ID; + + // ── CDI-injected direct OS handle (bypasses the phase router) ──────────── + @Inject + private OSIndexAPIImpl osIndexAPI; + + // ── Saved DB state — restored in @After to avoid polluting the running app ── + @SuppressWarnings("deprecation") + private IndiciesInfo savedEsInfo; + private Optional
Skip: when ES and OS point to the same cluster — in that case T0 exists in + * both client views so neither throws {@code index_not_found} and the + * fire-and-forget path is not exercised.
+ */ + @Test + public void test_closeIndex_phase1_osIndexNotFound_isSwallowed() { + assumeFalse("Requires separate ES and OS clusters to get real index_not_found from OS", + esSameAsOs()); + setPhase(1); + + try { + APILocator.getESIndexAPI().closeIndex(ES_WORKING); + Logger.info(this, "✅ closeIndex Phase 1 — OS index_not_found swallowed successfully"); + } catch (final RuntimeException e) { + throw new AssertionError( + "closeIndex must NOT throw in Phase 1 when OS has a different index name. Got: " + + e.getMessage(), e); + } + } + + /** + * Given Scenario: Phase 1. ES has T0; OS has T1. Caller uses the OS name T1. + * When : closeIndex("working_T1") is called — T1 exists in OS but NOT in ES. + * Then : ES is primary in Phase 1 → ES throws {@code index_not_found} for T1. + * Primary failure PROPAGATES — callers must not use the OS-native name in Phase 1. + * + *Skip: when ES and OS point to the same cluster — T1 exists in the ES view too.
+ */ + @Test(expected = RuntimeException.class) + public void test_closeIndex_phase1_esIndexNotFound_propagates() { + assumeFalse("Requires separate ES and OS clusters", esSameAsOs()); + setPhase(1); + + // OS_WORKING exists in OS but NOT in ES — ES is primary → must throw + APILocator.getESIndexAPI().closeIndex(OS_WORKING); + } + + /** + * Given Scenario: Phase 3 (OS only). OS has T1; T0 does not exist in OS. + * When : closeIndex("working_T0") is called. + * Then : OS is the only provider and is primary → exception propagates. + * + *Skip: when ES and OS point to the same cluster — T0 also exists in the OS view.
+ */ + @Test(expected = RuntimeException.class) + public void test_closeIndex_phase3_osIndexNotFound_propagates() { + assumeFalse("Requires separate ES and OS clusters", esSameAsOs()); + setPhase(3); + + APILocator.getESIndexAPI().closeIndex(ES_WORKING); + } + + // ========================================================================= + // deactivateIndex — ghost index (name never created in any cluster) + // + // deactivateIndex is a pure pointer-store operation: it does NOT query the cluster + // to verify the index exists. It uses IndexType.WORKING.is(name) (a startsWith check) + // to decide which DB slot to clear, then writes the result back to the store. + // + // Calling it with a name that was never created must: + // - not throw (cluster absence is irrelevant) + // - clear the working slot in the primary pointer store + // ========================================================================= + + /** + * Given Scenario: Phase 1 (dual-write, ES reads). {@code GHOST_WORKING} starts with + * "working" but was never created in either cluster. + * When : deactivateIndex(GHOST_WORKING) is called. + * Then : No exception — {@code deactivateIndex} never validates cluster existence. + * The ES DB working slot is cleared (name matched {@link IndexType#WORKING}). + * The OS DB working slot is also cleared (mirrored in Phase 1). + * + *This documents the fundamental contract: {@code deactivateIndex} is a + * pointer-store update driven by the name pattern, not by cluster state.
+ */ + @Test + @SuppressWarnings("deprecation") + public void test_deactivateIndex_ghostIndex_phase1_clearsWorkingSlot() + throws DotDataException, IOException { + setPhase(1); + + // Verify the ghost index truly doesn't exist in either cluster + assertFalse("Pre: ghost must not exist in ES", esImpl().indexExists(GHOST_WORKING)); + assertFalse("Pre: ghost must not exist in OS", osIndexAPI.indexExists(GHOST_WORKING)); + + // Give BOTH working AND live slots a known value. + // The live slot must survive the deactivate call so that the VersionedIndices + // builder always has at least one field set (saveIndices rejects empty builders). + contentletIndexAPI().activateIndex(ES_WORKING); + contentletIndexAPI().activateIndex(ES_LIVE); + assertNotNull("Pre: ES DB working slot must be non-null before deactivate", + APILocator.getIndiciesAPI().loadIndicies().getWorking()); + + // deactivateIndex with a ghost name must NOT throw + contentletIndexAPI().deactivateIndex(GHOST_WORKING); + + // ES DB working slot must be null — GHOST_WORKING starts with "working", + // so IndexType.WORKING.is(GHOST_WORKING) == true → builder.setWorking(null) + final IndiciesInfo esInfo = APILocator.getIndiciesAPI().loadIndicies(); + assertNull("ES DB working slot must be cleared after deactivating a ghost index", + esInfo.getWorking()); + + // OS DB working slot must also be cleared (mirrored in Phase 1). + // The live slot in OS must be preserved (builder kept it because IndexType.LIVE + // did not match GHOST_WORKING, so the save is valid). + final OptionalTests that require two physically separate clusters use this to skip themselves + * via {@link org.junit.Assume#assumeFalse}.
+ */ + private static boolean esSameAsOs() { + final String esEndpoint = Config.getStringProperty("DOT_ES_ENDPOINTS", + "http://localhost:9207"); + final String osEndpoint = Config.getStringProperty("OS_ENDPOINTS", + "http://localhost:9201"); + return esEndpoint.equalsIgnoreCase(osEndpoint.trim()); + } + + private static void setPhase(final int ordinal) { + Config.setProperty(FLAG_KEY, String.valueOf(ordinal)); + } + + private static ContentletIndexAPI contentletIndexAPI() { + return APILocator.getContentletIndexAPI(); + } + + private static ESIndexAPI esImpl() { + return ((IndexAPIImpl) APILocator.getESIndexAPI()).esImpl(); + } + + private void cleanupTestIndices() { + final ESIndexAPI esIndex = esImpl(); + for (final String name : List.of(ES_WORKING, ES_LIVE, DUAL_WORKING)) { + Try.run(() -> { if (esIndex.indexExists(name)) esIndex.delete(name); }) + .onFailure(e -> Logger.warn(this, + "Cleanup: error removing ES index '" + name + "': " + e.getMessage())); + } + for (final String name : List.of(OS_WORKING, OS_LIVE, DUAL_WORKING)) { + Try.run(() -> { if (osIndexAPI.indexExists(name)) osIndexAPI.delete(name); }) + .onFailure(e -> Logger.warn(this, + "Cleanup: error removing OS index '" + name + "': " + e.getMessage())); + } + } +}