fix(opensearch): make shadow write log level configurable (#35302)

dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java

@@ -4,6 +4,7 @@
 import static com.dotcms.content.index.IndexConfigHelper.isMigrationNotStarted;
 import static com.dotcms.content.index.IndexConfigHelper.isMigrationStarted;
 import static com.dotcms.content.index.IndexConfigHelper.isReadEnabled;
+import static com.dotcms.content.index.IndexConfigHelper.logShadowWriteFailure;
 import static com.dotmarketing.util.StringUtils.builder;
 
 import com.dotcms.api.system.event.message.MessageSeverity;
@@ -175,7 +176,8 @@ public ContentletIndexAPIImpl() {
              CDIUtils.getBeanThrows(ContentletIndexOperationsOS.class));
     }
 
-    /** Package-private constructor for testing. */
+    /** Package-private constructor for testing: injects only the two provider operations.
+     *  Still calls APILocator for the remaining dependencies. */
     ContentletIndexAPIImpl(final ContentletIndexOperations operationsES,
             final ContentletIndexOperations operationsOS) {
         this.operationsES = operationsES;
@@ -191,6 +193,31 @@ public ContentletIndexAPIImpl() {
         // Use getMappingAPI() for lazy initialization at first use.
     }
 
+    /**
+     * Full constructor for unit testing — injects all dependencies without calling
+     * {@link com.dotmarketing.business.APILocator}, allowing fully isolated tests.
+     *
+     * @param operationsES  ES write operations provider
+     * @param operationsOS  OS write operations provider
+     * @param indexAPI       phase-aware index management API (controls list/cluster operations)
+     * @param legacyIndiciesAPI  ES index pointer store (working/live slots)
+     * @param versionedIndicesAPI  OS index pointer store (working/live slots)
+     */
+    ContentletIndexAPIImpl(
+            final ContentletIndexOperations operationsES,
+            final ContentletIndexOperations operationsOS,
+            final IndexAPI indexAPI,
+            final IndiciesAPI legacyIndiciesAPI,
+            final VersionedIndicesAPI versionedIndicesAPI) {
+        this.operationsES       = operationsES;
+        this.operationsOS       = operationsOS;
+        this.router             = new PhaseRouter<>(operationsES, operationsOS);
+        this.queueApi           = null; // not needed for the methods under test
+        this.indexAPI           = indexAPI;
+        this.legacyIndiciesAPI  = legacyIndiciesAPI;
+        this.versionedIndicesAPI = versionedIndicesAPI;
+    }
+
     /**
      * Lazy initializer avoids circular reference Stackoverflow error.
      * Thread-safe: uses {@link AtomicReference#updateAndGet} to ensure
@@ -371,7 +398,7 @@ public void close() throws Exception {
                 } catch (final Exception e) {
                     if (entry.shadow) {
                         // OS shadow write — fire-and-forget: log divergence, do not propagate.
-                        Logger.warnAndDebug(CompositeBulkProcessor.class,
+                        logShadowWriteFailure(CompositeBulkProcessor.class,
                                 "OS shadow processor failed to flush on close — ES flush succeeded; "
                                         + "OS index may diverge until next reindex. Cause: "
                                         + e.getMessage(), e);
@@ -567,7 +594,9 @@ public synchronized boolean createContentIndex(final String indexName, final int
                 result = false;
             }
         }
-        MappingHelper.getInstance().addCustomMapping(indexName);
+        if (result) {
+            MappingHelper.getInstance().addCustomMapping(indexName);
+        }
         return result;
     }
 

dotCMS/src/main/java/com/dotcms/content/index/IndexConfigHelper.java

@@ -3,6 +3,7 @@
 import com.dotcms.content.index.opensearch.OSIndexProperty;
 import com.dotcms.featureflag.FeatureFlagName;
 import com.dotmarketing.util.Config;
+import com.dotmarketing.util.Logger;
 
 /**
  * Central helper for reading index-layer configuration properties.
@@ -35,6 +36,35 @@
  */
 public interface IndexConfigHelper {
 
+    /**
+     * Config key controlling the log level for OS shadow write failures in dual-write phases.
+     *
+     * <p>Valid values: {@code DEBUG}, {@code INFO}, {@code WARN}, {@code ERROR} (default: {@code WARN}).
+     * Set to {@code ERROR} or {@code DEBUG} to increase/decrease visibility during migration QA.</p>
+     */
+    String SHADOW_WRITE_LOG_LEVEL_KEY = "DOTCMS_SHADOW_WRITE_LOG_LEVEL";
+
+    /**
+     * Logs an OS shadow write failure at the level configured by
+     * {@value #SHADOW_WRITE_LOG_LEVEL_KEY} (default: {@code WARN}).
+     *
+     * @param clazz   the class to attribute the log entry to
+     * @param message the log message
+     * @param t       the throwable, or {@code null} if none
+     */
+    static void logShadowWriteFailure(final Class<?> clazz,
+                                      final String message,
+                                      final Throwable t) {
+        final String level = Config.getStringProperty(SHADOW_WRITE_LOG_LEVEL_KEY, "WARN")
+                                   .toUpperCase();
+        switch (level) {
+            case "DEBUG": Logger.debug(clazz, message, t); break;
+            case "INFO":  Logger.info(clazz,  message);    break;
+            case "ERROR": Logger.error(clazz, message, t); break;
+            default:      Logger.warn(clazz,  message, t); break;
+        }
+    }
+
     // -------------------------------------------------------------------------
     // Migration phase
     // -------------------------------------------------------------------------

dotCMS/src/main/java/com/dotcms/content/index/PhaseRouter.java

@@ -4,6 +4,8 @@
 import static com.dotcms.content.index.IndexConfigHelper.isMigrationNotStarted;
 import static com.dotcms.content.index.IndexConfigHelper.isReadEnabled;
 
+import static com.dotcms.content.index.IndexConfigHelper.logShadowWriteFailure;
+
 import com.dotmarketing.util.Logger;
 import java.util.List;
 import java.util.function.Consumer;
@@ -246,7 +248,7 @@ public boolean writeBoolean(final Function<T, Boolean> fn) {
                 if (impl == primary) {
                     primaryEx = e;
                 } else {
-                    Logger.warn(PhaseRouter.class,
+                    logShadowWriteFailure(PhaseRouter.class,
                             "Shadow write failed (fire-and-forget in dual-write phase): "
                             + e.getMessage(), e);
                 }
@@ -332,7 +334,7 @@ public void writeChecked(final ThrowingConsumer<T> action) throws Exception {
                 if (impl == primary) {
                     primaryEx = e;  // record — shadow must still be called
                 } else {
-                    Logger.warn(PhaseRouter.class,
+                    logShadowWriteFailure(PhaseRouter.class,
                             "Shadow write failed (fire-and-forget in dual-write phase): "
                             + e.getMessage(), e);
                 }
@@ -372,7 +374,7 @@ public <R> R writeReturningChecked(final ThrowingFunction<T, R> fn) throws Excep
         try {
             fn.apply(shadow);
         } catch (Exception e) {
-            Logger.warn(PhaseRouter.class,
+            logShadowWriteFailure(PhaseRouter.class,
                     "Shadow write failed (fire-and-forget in dual-write phase): "
                     + e.getMessage(), e);
         }

dotCMS/src/main/java/com/dotmarketing/common/reindex/BulkProcessorListener.java

@@ -1,5 +1,7 @@
 package com.dotmarketing.common.reindex;
 
+import static com.dotcms.content.index.IndexConfigHelper.logShadowWriteFailure;
+
 import com.dotcms.content.index.IndexConfigHelper;
 import com.dotcms.content.index.IndexTag;
 import com.dotcms.content.index.domain.IndexBulkItemResult;
@@ -111,8 +113,8 @@ public void afterBulk(final long executionId, final List<IndexBulkItemResult> re
             // OS shadow — fire-and-forget; log individual failures for observability only
             results.stream()
                     .filter(IndexBulkItemResult::failed)
-                    .forEach(r -> Logger.warn(this.getClass(),
-                            "[OS] Index failure (fire-and-forget): " + r.failureMessage()));
+                    .forEach(r -> logShadowWriteFailure(this.getClass(),
+                            "[OS] Index failure (fire-and-forget): " + r.failureMessage(), null));
             return;
         }
         Logger.debug(this.getClass(), "Bulk process completed");
@@ -155,7 +157,7 @@ public void afterBulk(final long executionId, final List<IndexBulkItemResult> re
     public void afterBulk(final long executionId, final Throwable failure) {
         final String msg = failure != null ? failure.getMessage() : "(no message)";
         if (shadow) {
-            Logger.warnAndDebug(this.getClass(),
+            logShadowWriteFailure(this.getClass(),
                     "[OS] Bulk process failed entirely (fire-and-forget): " + msg, failure);
             return;
         }

dotCMS/src/main/resources/dotmarketing-config.properties

@@ -922,3 +922,14 @@ telemetry.collection.timeout.seconds=30
 # Metrics taking longer than this will be logged as warnings for optimization
 # Default: 500 milliseconds
 telemetry.metric.slow.threshold.ms=500
+
+## OpenSearch migration — shadow write observability
+# Log level for OS shadow write failures during dual-write phases (1 and 2).
+# Shadow failures are fire-and-forget: the ES (primary) result is returned to the
+# caller regardless of OS outcome. This setting controls how loudly those failures
+# are reported.
+# Valid values: DEBUG, INFO, WARN, ERROR  (default: WARN)
+# Set to ERROR to surface shadow failures in dashboards/alerts.
+# Set to DEBUG to suppress them during steady-state migration.
+#DOTCMS_SHADOW_WRITE_LOG_LEVEL=WARN
+