RANGER-5743: Docker setup for Apache Ranger ElasticSearch plugin and Integrate ES plugin with x-pack-security on ES 7.17

agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java

@@ -20,6 +20,7 @@
 
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.ranger.audit.destination.AuditDestination;
+import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.queue.AuditAsyncQueue;
 import org.apache.ranger.audit.queue.AuditBatchQueue;
 import org.apache.ranger.audit.queue.AuditFileQueue;
@@ -77,7 +78,8 @@ public class AuditProviderFactory {
     private       String            componentAppType = "";
     private       boolean           mInitDone;
     private       JVMShutdownHook   jvmShutdownHook;
-    private final ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional"));
+    private final List<AuditHandler> requestThreadDestinations = new ArrayList<>();
+    private final ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional", "elasticsearch"));
 
     public AuditProviderFactory() {
         LOG.info("AuditProviderFactory: creating..");
@@ -107,6 +109,27 @@ public AuditHandler getAuditProvider() {
         return mProvider;
     }
 
+    /**
+     * Delivers an audit event directly to configured destinations on the calling thread,
+     * bypassing async/batch queue threads. Used by the Elasticsearch plugin where ES Security
+     * Manager grants network permissions only to the request thread.
+     */
+    public boolean logOnRequestThread(AuditEventBase event) {
+        if (event == null || requestThreadDestinations.isEmpty()) {
+            return false;
+        }
+
+        boolean ret = true;
+
+        for (AuditHandler handler : requestThreadDestinations) {
+            if (!handler.log(event)) {
+                ret = false;
+            }
+        }
+
+        return ret;
+    }
+
     public boolean isInitDone() {
         return mInitDone;
     }
@@ -128,6 +151,8 @@ public synchronized void init(Properties props, String appType) {
             LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
         }
 
+        requestThreadDestinations.clear();
+
         mInitDone        = true;
         componentAppType = appType;
 
@@ -187,6 +212,7 @@ public synchronized void init(Properties props, String appType) {
 
             if (destProvider != null) {
                 destProvider.init(props, destPropPrefix);
+                requestThreadDestinations.add(destProvider);
 
                 String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE);
 

agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java

@@ -72,13 +72,6 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId,
 
         addResourcesForServiceType(serviceType);
 
-        this.serviceType    = serviceType;
-        this.appId          = StringUtils.isEmpty(appId) ? serviceType : appId;
-        this.propertyPrefix = "ranger.plugin." + serviceType;
-        this.serviceName    = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName;
-
-        addResourcesForServiceName(this.serviceType, this.serviceName);
-
         if (additionalConfigFiles != null) {
             for (File configFile : additionalConfigFiles) {
                 try {
@@ -89,6 +82,13 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId,
             }
         }
 
+        this.serviceType    = serviceType;
+        this.appId          = StringUtils.isEmpty(appId) ? serviceType : appId;
+        this.propertyPrefix = "ranger.plugin." + serviceType;
+        this.serviceName    = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName;
+
+        addResourcesForServiceName(this.serviceType, this.serviceName);
+
         String trustedProxyAddressString = this.get(propertyPrefix + ".trusted.proxy.ipaddresses");
 
         if (StringUtil.isEmpty(clusterName)) {

agents-common/src/test/java/org/apache/ranger/authorization/hadoop/config/TestRangerPluginConfig.java

@@ -79,7 +79,7 @@ public void test02_constructor_serviceNameFromConfig_whenNullInCtor() throws Exc
 
         RangerPluginConfig cfg = new RangerPluginConfig("hdfs", null, null, null, null, files, null);
 
-        assertNull(cfg.getServiceName());
+        assertEquals("svcA", cfg.getServiceName());
         assertEquals("svcA", cfg.get(key));
     }
 

audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java

@@ -58,13 +58,29 @@
 public class AuditREST {
     private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class);
 
-    private static final Map<String, Set<String>> allowedServiceUsers;
+    private static volatile Map<String, Set<String>> allowedServiceUsers;
 
     static {
-        allowedServiceUsers = initializeAllowedUsers();
         initializeAuthToLocal();
     }
 
+    private static Map<String, Set<String>> getAllowedServiceUsers() {
+        Map<String, Set<String>> ret = allowedServiceUsers;
+
+        if (ret == null) {
+            synchronized (AuditREST.class) {
+                ret = allowedServiceUsers;
+
+                if (ret == null) {
+                    allowedServiceUsers = initializeAllowedUsers();
+                    ret               = allowedServiceUsers;
+                }
+            }
+        }
+
+        return ret;
+    }
+
     @Autowired
     AuditDestinationMgr auditDestinationMgr;
 
@@ -330,7 +346,7 @@ private boolean isAllowedServiceUser(String serviceName, String userName) {
         boolean ret;
 
         if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) {
-            Set<String> allowedUsers = allowedServiceUsers.get(serviceName);
+            Set<String> allowedUsers = getAllowedServiceUsers().get(serviceName);
 
             ret = allowedUsers != null && allowedUsers.contains(userName);
         } else {

audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml

@@ -242,13 +242,20 @@
         <description>Allowed users for dev_solr (Solr plugin)</description>
     </property>
 
+    <property>
+        <name>ranger.audit.ingestor.service.dev_elasticsearch.allowed.users</name>
+        <value>elasticsearch</value>
+        <description>Allowed users for dev_elasticsearch (Elasticsearch plugin)</description>
+    </property>
+
     <!-- AUTH_TO_LOCAL RULES FOR KERBEROS PRINCIPAL MAPPING -->
     <property>
         <name>ranger.audit.ingestor.auth.to.local</name>
         <value>
                RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/
                RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/
                RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/
+               RULE:[2:$1/$2@$0](elasticsearch/.*@.*)s/.*/elasticsearch/
                RULE:[1:$1@$0](.*@.*)s/@.*//
                DEFAULT
         </value>

dev-support/ranger-docker/.dockerignore

@@ -17,5 +17,6 @@
 !dist/ranger-*-trino-plugin.tar.gz
 !dist/ranger-*-ozone-plugin.tar.gz
 !dist/ranger-*-solr-plugin.tar.gz
+!dist/ranger-*-elasticsearch-plugin.tar.gz
 !downloads/*
 !scripts/*

dev-support/ranger-docker/.env

@@ -88,6 +88,13 @@ TRINO_VERSION=latest
 # Open Search
 OPENSEARCH_VERSION=1.3.19
 
+# Elasticsearch Configuration (authorization plugin testing)
+ELASTICSEARCH_VERSION=7.17.29
+ELASTICSEARCH_PLUGIN_VERSION=3.0.0-SNAPSHOT
+ELASTICSEARCH_BOOTSTRAP_PASSWORD=rangerR0cks!
+RANGER_ADMIN_USER=admin
+RANGER_ADMIN_PASSWORD=rangerR0cks!
+
 # Debug Configuration
 DEBUG_ADMIN=false
 DEBUG_USERSYNC=false

dev-support/ranger-docker/Dockerfile.ranger-elasticsearch

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE=apache/ranger-base
+ARG RANGER_BASE_VERSION=20260123-2-8
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+ARG ELASTICSEARCH_VERSION
+ARG ELASTICSEARCH_PLUGIN_VERSION
+
+COPY ./dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz   /home/ranger/dist/
+COPY ./scripts/elasticsearch/*.sh                                             ${RANGER_SCRIPTS}/
+COPY ./scripts/elasticsearch/patch-ranger-security-xml.py                     ${RANGER_SCRIPTS}/
+COPY ./scripts/elasticsearch/patch-ranger-audit-xml.py                        ${RANGER_SCRIPTS}/
+COPY ./downloads/jaxb-api-2.2.11.jar                                          ${RANGER_SCRIPTS}/elasticsearch-lib/
+COPY ./downloads/jaxb-runtime-2.3.2.jar                                       ${RANGER_SCRIPTS}/elasticsearch-lib/
+COPY ./downloads/javax.activation-api-1.2.0.jar                               ${RANGER_SCRIPTS}/elasticsearch-lib/
+
+RUN groupadd -r elasticsearch 2>/dev/null || true && \
+    useradd -r -g hadoop -d /opt/elasticsearch -s /bin/bash elasticsearch 2>/dev/null || true && \
+    tar xvfz /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz --directory=/opt/ && \
+    ln -s /opt/elasticsearch-${ELASTICSEARCH_VERSION} /opt/elasticsearch && \
+    rm -f /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz && \
+    mkdir -p /opt/elasticsearch/data /opt/elasticsearch/logs && \
+    tar xvfz /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz --directory=/opt/ranger && \
+    ln -s /opt/ranger/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin /opt/ranger/ranger-elasticsearch-plugin && \
+    rm -f /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz && \
+    rm -f /opt/ranger/ranger-elasticsearch-plugin/install.properties && \
+    chown -R elasticsearch:hadoop /opt/elasticsearch* && \
+    chmod 744 ${RANGER_SCRIPTS}/ranger-elasticsearch-setup.sh \
+              ${RANGER_SCRIPTS}/ranger-elasticsearch-post-setup.sh \
+              ${RANGER_SCRIPTS}/ranger-elasticsearch.sh
+
+ENV ELASTICSEARCH_HOME=/opt/elasticsearch
+ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/elasticsearch/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-elasticsearch.sh" ]

dev-support/ranger-docker/README.md

@@ -34,7 +34,7 @@ Use Dockerfiles in this directory to create docker images and run them to build
    ~~~
    chmod +x download-archives.sh
    # use a subset of the below to download specific services
-   ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch
+   ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch elasticsearch
    ~~~
 
 - Execute following commands to set environment variables to build Apache Ranger docker containers:
@@ -157,6 +157,41 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml up
 docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-opensearch.yml up -d
 ~~~
 
+#### Bring up elasticsearch container (authorization plugin testing):
+~~~
+# Prerequisites: build Ranger artifacts and download the Elasticsearch archive
+mvn clean package -pl distro -am -DskipTests
+cp target/ranger-* dev-support/ranger-docker/dist/
+cd dev-support/ranger-docker
+./download-archives.sh elasticsearch
+
+export RANGER_DB_TYPE=postgres
+
+# Host port 9201 maps to container 9200 (avoids conflict with OpenSearch on 9200).
+# On Linux, ensure vm.max_map_count >= 262144 (e.g. sudo sysctl -w vm.max_map_count=262144).
+docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml \
+  -f docker-compose.ranger-elasticsearch.yml up -d --build
+~~~
+
+Elasticsearch starts with X-Pack Security enabled (native realm). Default credentials:
+
+- `elastic` / value of `ELASTICSEARCH_BOOTSTRAP_PASSWORD` in `.env` (default: `rangerR0cks!`)
+- `testuser_2` / same password (created automatically for authorization testing)
+
+Smoke tests after the container is healthy:
+
+~~~
+# Authenticated request (expect 200 or policy-based 403, not 401)
+curl -u elastic:rangerR0cks! http://localhost:9201/test-index/_search
+
+# Unauthenticated request (expect 401)
+curl http://localhost:9201/test-index/_search
+~~~
+
+Ranger Admin registers the `dev_elasticsearch` service automatically on first startup.
+The plugin polls policies from Ranger Admin; allow up to 30 seconds after startup for
+policy refresh before running authorization tests.
+
 #### OpenSearch audit flow (replace Solr for access audits)
 
 OpenSearch can replace Solr for **audit storage and UI queries**. Ranger Admin reads audits via

dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml

@@ -0,0 +1,58 @@
+services:
+  ranger-elasticsearch:
+    build:
+      context: .
+      dockerfile: Dockerfile.ranger-elasticsearch
+      args:
+        - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+        - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+        - ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION}
+        - ELASTICSEARCH_PLUGIN_VERSION=${ELASTICSEARCH_PLUGIN_VERSION}
+    image: ranger-elasticsearch
+    container_name: ranger-elasticsearch
+    hostname: ranger-elasticsearch.rangernw
+    volumes:
+      - ./dist/version:/home/ranger/dist/version:ro
+      - ./dist/keytabs/ranger-elasticsearch:/etc/keytabs
+      - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro
+      - ./scripts/elasticsearch/elasticsearch.yml:/home/ranger/scripts/elasticsearch.yml:ro
+      - ./scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties:/opt/ranger/ranger-elasticsearch-plugin/install.properties
+      - ./scripts/elasticsearch/patch-ranger-audit-xml.py:/home/ranger/scripts/patch-ranger-audit-xml.py:ro
+      - ./scripts/elasticsearch/ranger-elasticsearch.sh:/home/ranger/scripts/ranger-elasticsearch.sh:ro
+      - elasticsearch-data:/opt/elasticsearch/data
+      - elasticsearch-logs:/opt/elasticsearch/logs
+    stdin_open: true
+    tty: true
+    networks:
+      - ranger
+    ports:
+      - "9201:9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    depends_on:
+      ranger-kdc:
+        condition: service_started
+      ranger:
+        condition: service_started
+    environment:
+      - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+      - KRB5_CONFIG=/etc/krb5.conf
+      - ELASTICSEARCH_VERSION
+      - ELASTICSEARCH_PLUGIN_VERSION
+      - ELASTICSEARCH_BOOTSTRAP_PASSWORD
+      - RANGER_ADMIN_USER
+      - RANGER_ADMIN_PASSWORD
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+
+volumes:
+  elasticsearch-data:
+  elasticsearch-logs:
+
+networks:
+  ranger:
+    name: rangernw

dev-support/ranger-docker/download-archives.sh

@@ -17,7 +17,7 @@
 # limitations under the License.
 
 #
-# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone archives to a local cache directory.
+# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone/OpenSearch/Elasticsearch archives to a local cache directory.
 # The downloaded archives will be used while building docker images that run these services.
 #
 
@@ -139,6 +139,7 @@ then
     downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz            https://archive.apache.org/dist/knox/${KNOX_VERSION}
     extractOzoneIfNeeded
     downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}
+    downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch
 else
   for arg in "$@"; do
     if [[ $arg == 'hadoop' ]]
@@ -164,6 +165,12 @@ else
     elif [[ $arg == 'opensearch' ]]
     then
       downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}
+    elif [[ $arg == 'elasticsearch' ]]
+    then
+      downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch
+      downloadIfNotPresent jaxb-api-2.2.11.jar https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.2.11
+      downloadIfNotPresent jaxb-runtime-2.3.2.jar https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/2.3.2
+      downloadIfNotPresent javax.activation-api-1.2.0.jar https://repo1.maven.org/maven2/javax/activation/javax.activation-api/1.2.0
     else
       echo "Passed argument $arg is invalid!"
     fi