From 298153a487a741eed2365da28b16e9d089b661a6 Mon Sep 17 00:00:00 2001 From: Pradeep Agrawal Date: Wed, 12 Aug 2026 12:53:20 +0530 Subject: [PATCH 1/2] RANGER-5740: Align Elasticsearch plugin caller identity with X-Pack Security --- ranger-elasticsearch-plugin-shim/pom.xml | 36 +++++++++ .../plugin/RangerElasticsearchPlugin.java | 4 +- .../filter/RangerSecurityActionFilter.java | 23 +++++- ...lasticsearchAuthenticatedUserResolver.java | 56 +++++++++++++ .../authc/user/UsernamePasswordToken.java | 8 ++ .../rest/filter/RangerSecurityRestFilter.java | 27 ++++--- ...lasticsearchAuthenticatedUserResolver.java | 65 +++++++++++++++ .../filter/TestRangerSecurityRestFilter.java | 79 +++++++++++++++++++ 8 files changed, 282 insertions(+), 16 deletions(-) create mode 100644 ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java create mode 100644 ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java create mode 100644 ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/TestRangerSecurityRestFilter.java diff --git a/ranger-elasticsearch-plugin-shim/pom.xml b/ranger-elasticsearch-plugin-shim/pom.xml index 9dedb37d117..eaa25d71e66 100644 --- a/ranger-elasticsearch-plugin-shim/pom.xml +++ b/ranger-elasticsearch-plugin-shim/pom.xml @@ -70,5 +70,41 @@ + + org.elasticsearch.plugin + x-pack-core + ${elasticsearch.version} + provided + + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + test + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + test + + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java index aa1c10a30f8..e3b740f00b8 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java @@ -72,7 +72,7 @@ public List getActionFilters() { @Override public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { - return handler -> new RangerSecurityRestFilter(threadContext, handler); + return handler -> new RangerSecurityRestFilter(settings, threadContext, handler); } @Override @@ -81,7 +81,7 @@ public Collection createComponents(final Client client, final ClusterSer final NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { addPluginConfig2Classpath(environment); - rangerSecurityActionFilter = new RangerSecurityActionFilter(threadPool.getThreadContext()); + rangerSecurityActionFilter = new RangerSecurityActionFilter(settings, threadPool.getThreadContext()); return Collections.singletonList(rangerSecurityActionFilter); } diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java index 31af2e3a414..1d70e3b0a1c 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer; +import org.apache.ranger.authorization.elasticsearch.plugin.authc.ElasticsearchAuthenticatedUserResolver; import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken; import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils; import org.elasticsearch.ElasticsearchStatusException; @@ -28,6 +29,7 @@ import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -39,12 +41,14 @@ public class RangerSecurityActionFilter extends AbstractLifecycleComponent implements ActionFilter { private static final Logger LOG = LoggerFactory.getLogger(RangerSecurityActionFilter.class); + private final Settings settings; private final ThreadContext threadContext; private final RangerElasticsearchAuthorizer rangerElasticsearchAuthorizer = new RangerElasticsearchAuthorizer(); - public RangerSecurityActionFilter(ThreadContext threadContext) { + public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext) { super(); + this.settings = settings; this.threadContext = threadContext; } @@ -57,7 +61,18 @@ public int order() { public void apply(Task task, String action, Request request, ActionListener listener, ActionFilterChain chain) { String user = threadContext.getTransient(UsernamePasswordToken.USERNAME); - // If user is not null, then should check permission of the outside caller. + if (StringUtils.isEmpty(user)) { + ElasticsearchAuthenticatedUserResolver authResolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + if (authResolver.requiresAuthenticatedUser()) { + user = authResolver.resolveUsername(); + + if (StringUtils.isNotEmpty(user)) { + threadContext.putTransient(UsernamePasswordToken.USERNAME, user); + } + } + } + if (StringUtils.isNotEmpty(user)) { List indexs = RequestUtils.getIndexFromRequest(request); String clientIPAddress = threadContext.getTransient(RequestUtils.CLIENT_IP_ADDRESS); @@ -71,8 +86,10 @@ public void app throw new ElasticsearchStatusException(errorMsg, RestStatus.FORBIDDEN, user, action, index); } } + } else if (threadContext.isSystemContext()) { + LOG.debug("System context request, skipping Ranger permission check for action[{}].", action); } else { - LOG.debug("User is null, no check permission for elasticsearch do action[{}] with request[{}]", action, request); + throw new ElasticsearchStatusException("Error: Request requires authenticated user.", RestStatus.UNAUTHORIZED); } chain.proceed(task, action, request, listener); diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java new file mode 100644 index 00000000000..823d9502ef8 --- /dev/null +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.elasticsearch.plugin.authc; + +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.xpack.core.security.SecurityContext; +import org.elasticsearch.xpack.core.security.user.User; + +/** + * Resolves the Elasticsearch-verified user for the current request. + * Caller identity must come from X-Pack Security, not from client-supplied headers. + */ +public class ElasticsearchAuthenticatedUserResolver { + private final SecurityContext securityContext; + private final ThreadContext threadContext; + + public ElasticsearchAuthenticatedUserResolver(Settings settings, ThreadContext threadContext) { + this.securityContext = new SecurityContext(settings, threadContext); + this.threadContext = threadContext; + } + + public boolean requiresAuthenticatedUser() { + return !threadContext.isSystemContext(); + } + + public String resolveUsername() { + String username = null; + + if (requiresAuthenticatedUser()) { + User user = securityContext.getUser(); + + if (user != null) { + username = user.principal(); + } + } + + return StringUtils.isEmpty(username) ? null : username; + } +} diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/user/UsernamePasswordToken.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/user/UsernamePasswordToken.java index 958cda381d9..4b964f6a3f6 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/user/UsernamePasswordToken.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/user/UsernamePasswordToken.java @@ -28,6 +28,10 @@ import java.util.List; import java.util.Map; +/** + * Legacy helper for parsing Basic credentials from a REST request. + * Parsed credentials must not be used as an authenticated identity. + */ public class UsernamePasswordToken { public static final String USERNAME = "username"; public static final String BASIC_AUTH_PREFIX = "Basic "; @@ -41,6 +45,10 @@ public UsernamePasswordToken(String username, String password) { this.password = password; } + /** + * @deprecated Do not use parsed Basic credentials for authorization decisions. + */ + @Deprecated public static UsernamePasswordToken parseToken(RestRequest request) { Map> headers = request.getHeaders(); diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java index 09b15cdc20a..97a0234c415 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java @@ -18,11 +18,13 @@ package org.apache.ranger.authorization.elasticsearch.plugin.rest.filter; import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.authorization.elasticsearch.plugin.authc.ElasticsearchAuthenticatedUserResolver; import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken; import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestHandler; @@ -34,29 +36,32 @@ public class RangerSecurityRestFilter extends AbstractLifecycleComponent implements RestHandler { private static final Logger LOG = LoggerFactory.getLogger(RangerSecurityRestFilter.class); + private final Settings settings; private final RestHandler restHandler; private final ThreadContext threadContext; - public RangerSecurityRestFilter(final ThreadContext threadContext, final RestHandler restHandler) { + public RangerSecurityRestFilter(final Settings settings, final ThreadContext threadContext, final RestHandler restHandler) { super(); + this.settings = settings; this.restHandler = restHandler; this.threadContext = threadContext; } @Override public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception { - // Now only support to get user from request, - // it should work with other elasticsearch identity authentication plugins in fact. - UsernamePasswordToken user = UsernamePasswordToken.parseToken(request); - - if (user == null) { - throw new ElasticsearchStatusException("Error: User is null, the request requires user authentication.", RestStatus.UNAUTHORIZED); - } else { - LOG.debug("Success to parse user[{}] from request[{}].", user, request); - } + ElasticsearchAuthenticatedUserResolver authResolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + if (authResolver.requiresAuthenticatedUser()) { + String username = authResolver.resolveUsername(); - threadContext.putTransient(UsernamePasswordToken.USERNAME, user.getUsername()); + if (StringUtils.isEmpty(username)) { + throw new ElasticsearchStatusException("Error: Request requires authenticated user.", RestStatus.UNAUTHORIZED); + } + + threadContext.putTransient(UsernamePasswordToken.USERNAME, username); + LOG.debug("Using Elasticsearch-verified user[{}] for request[{}].", username, request); + } String clientIPAddress = RequestUtils.getClientIPAddress(request); diff --git a/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java new file mode 100644 index 00000000000..c3d0f9a4240 --- /dev/null +++ b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.elasticsearch.plugin.authc; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.xpack.core.security.SecurityContext; +import org.elasticsearch.xpack.core.security.user.User; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TestElasticsearchAuthenticatedUserResolver { + @Test + void resolveUsernameReturnsVerifiedUser() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + SecurityContext securityContext = new SecurityContext(settings, threadContext); + + securityContext.setUser(new User("admin"), Version.CURRENT); + + ElasticsearchAuthenticatedUserResolver resolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + Assertions.assertTrue(resolver.requiresAuthenticatedUser()); + Assertions.assertEquals("admin", resolver.resolveUsername()); + } + + @Test + void resolveUsernameReturnsNullWithoutVerifiedUser() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + ElasticsearchAuthenticatedUserResolver resolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + Assertions.assertTrue(resolver.requiresAuthenticatedUser()); + Assertions.assertNull(resolver.resolveUsername()); + } + + @Test + void systemContextDoesNotRequireAuthenticatedUser() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + + threadContext.markAsSystemContext(); + + ElasticsearchAuthenticatedUserResolver resolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + Assertions.assertFalse(resolver.requiresAuthenticatedUser()); + Assertions.assertNull(resolver.resolveUsername()); + } +} diff --git a/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/TestRangerSecurityRestFilter.java b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/TestRangerSecurityRestFilter.java new file mode 100644 index 00000000000..171861bc9e7 --- /dev/null +++ b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/TestRangerSecurityRestFilter.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.elasticsearch.plugin.rest.filter; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.HttpChannel; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.security.SecurityContext; +import org.elasticsearch.xpack.core.security.user.User; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TestRangerSecurityRestFilter { + @Test + void rejectsUnverifiedBasicCredentials() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + RestHandler restHandler = Mockito.mock(RestHandler.class); + RestRequest request = Mockito.mock(RestRequest.class); + RestChannel channel = Mockito.mock(RestChannel.class); + NodeClient client = Mockito.mock(NodeClient.class); + RangerSecurityRestFilter filter = new RangerSecurityRestFilter(settings, threadContext, restHandler); + + Mockito.when(request.getHeaders()).thenReturn(java.util.Collections.singletonMap( + "Authorization", java.util.Collections.singletonList("Basic YWRtaW46d3JvbmdwYXNzd29yZA=="))); + + ElasticsearchStatusException exception = Assertions.assertThrows( + ElasticsearchStatusException.class, + () -> filter.handleRequest(request, channel, client)); + + Assertions.assertEquals(RestStatus.UNAUTHORIZED, exception.status()); + Mockito.verifyNoInteractions(restHandler); + } + + @Test + void acceptsElasticsearchVerifiedUser() throws Exception { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + SecurityContext securityContext = new SecurityContext(settings, threadContext); + RestHandler restHandler = Mockito.mock(RestHandler.class); + RestRequest request = Mockito.mock(RestRequest.class); + RestChannel channel = Mockito.mock(RestChannel.class); + NodeClient client = Mockito.mock(NodeClient.class); + HttpChannel httpChannel = Mockito.mock(HttpChannel.class); + RangerSecurityRestFilter filter = new RangerSecurityRestFilter(settings, threadContext, restHandler); + + Mockito.when(request.getHttpChannel()).thenReturn(httpChannel); + Mockito.when(httpChannel.getRemoteAddress()).thenReturn(null); + + securityContext.setUser(new User("admin"), Version.CURRENT); + + filter.handleRequest(request, channel, client); + + Mockito.verify(restHandler).handleRequest(request, channel, client); + } +} From a13a75dcce47ac856cab0badf6442263be8c055a Mon Sep 17 00:00:00 2001 From: Pradeep Agrawal Date: Thu, 13 Aug 2026 16:05:47 +0530 Subject: [PATCH 2/2] RANGER-5743: Docker setup for Apache Ranger ElasticSearch plugin and Integrate ES plugin with x-pack-security on ES 7.17 Co-authored-by: Cursor --- .../audit/provider/AuditProviderFactory.java | 28 +- .../hadoop/config/RangerPluginConfig.java | 14 +- .../hadoop/config/TestRangerPluginConfig.java | 2 +- .../apache/ranger/audit/rest/AuditREST.java | 22 +- .../conf/ranger-audit-ingestor-site.xml | 7 + dev-support/ranger-docker/.dockerignore | 1 + dev-support/ranger-docker/.env | 7 + .../Dockerfile.ranger-elasticsearch | 51 +++ dev-support/ranger-docker/README.md | 37 +- .../docker-compose.ranger-elasticsearch.yml | 58 +++ .../ranger-docker/download-archives.sh | 9 +- .../scripts/admin/create-ranger-services.py | 17 +- .../scripts/elasticsearch/elasticsearch.yml | 27 ++ .../elasticsearch/patch-ranger-audit-xml.py | 69 ++++ .../patch-ranger-security-xml.py | 71 ++++ ...er-elasticsearch-plugin-install.properties | 102 +++++ .../ranger-elasticsearch-post-setup.sh | 173 +++++++++ .../ranger-elasticsearch-setup.sh | 97 +++++ .../elasticsearch/ranger-elasticsearch.sh | 78 ++++ .../scripts/kafka/ranger-kafka-setup.sh | 6 +- .../ranger-docker/scripts/kdc/entrypoint.sh | 4 +- distro/pom.xml | 24 ++ distro/src/main/assembly/pdp.xml | 13 + .../main/assembly/plugin-elasticsearch.xml | 114 +++--- distro/src/main/assembly/plugin-solr.xml | 8 + .../ranger-elasticsearch-audit-changes.cfg | 1 + plugin-elasticsearch/pom.xml | 17 + .../scripts/install.properties | 1 + .../ElasticsearchAuditIngestorClient.java | 348 ++++++++++++++++++ .../RangerElasticsearchAuditHandler.java | 59 ++- .../RangerElasticsearchAuthorizer.java | 30 +- .../RangerServiceElasticsearch.java | 2 +- pom.xml | 1 + .../conf/plugin-descriptor.properties | 5 +- .../conf/plugin-security.policy | 49 ++- .../RangerElasticsearchAuthorizer.java | 102 ----- ...RangerElasticsearchAuthorizerDelegate.java | 94 +++++ .../plugin/RangerElasticsearchPlugin.java | 73 ++-- .../filter/RangerSecurityActionFilter.java | 55 ++- ...lasticsearchAuthenticatedUserResolver.java | 29 ++ ...lasticsearchAuthenticatedUserResolver.java | 24 ++ ranger_in_docker | 2 +- 42 files changed, 1662 insertions(+), 269 deletions(-) create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-elasticsearch create mode 100644 dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml create mode 100644 dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml create mode 100644 dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py create mode 100644 dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py create mode 100644 dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties create mode 100755 dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh create mode 100755 dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh create mode 100755 dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh create mode 100644 plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java delete mode 100644 ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java create mode 100644 ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index ec42d89f153..16c012f9b1e 100644 --- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -20,6 +20,7 @@ import org.apache.hadoop.util.ShutdownHookManager; import org.apache.ranger.audit.destination.AuditDestination; +import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.queue.AuditAsyncQueue; import org.apache.ranger.audit.queue.AuditBatchQueue; import org.apache.ranger.audit.queue.AuditFileQueue; @@ -77,7 +78,8 @@ public class AuditProviderFactory { private String componentAppType = ""; private boolean mInitDone; private JVMShutdownHook jvmShutdownHook; - private final ArrayList hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional")); + private final List requestThreadDestinations = new ArrayList<>(); + private final ArrayList hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional", "elasticsearch")); public AuditProviderFactory() { LOG.info("AuditProviderFactory: creating.."); @@ -107,6 +109,27 @@ public AuditHandler getAuditProvider() { return mProvider; } + /** + * Delivers an audit event directly to configured destinations on the calling thread, + * bypassing async/batch queue threads. Used by the Elasticsearch plugin where ES Security + * Manager grants network permissions only to the request thread. + */ + public boolean logOnRequestThread(AuditEventBase event) { + if (event == null || requestThreadDestinations.isEmpty()) { + return false; + } + + boolean ret = true; + + for (AuditHandler handler : requestThreadDestinations) { + if (!handler.log(event)) { + ret = false; + } + } + + return ret; + } + public boolean isInitDone() { return mInitDone; } @@ -128,6 +151,8 @@ public synchronized void init(Properties props, String appType) { LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize"); } + requestThreadDestinations.clear(); + mInitDone = true; componentAppType = appType; @@ -187,6 +212,7 @@ public synchronized void init(Properties props, String appType) { if (destProvider != null) { destProvider.init(props, destPropPrefix); + requestThreadDestinations.add(destProvider); String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE); diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java index c5ee9e54d56..0e1a57f7fac 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java @@ -72,13 +72,6 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId, addResourcesForServiceType(serviceType); - this.serviceType = serviceType; - this.appId = StringUtils.isEmpty(appId) ? serviceType : appId; - this.propertyPrefix = "ranger.plugin." + serviceType; - this.serviceName = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName; - - addResourcesForServiceName(this.serviceType, this.serviceName); - if (additionalConfigFiles != null) { for (File configFile : additionalConfigFiles) { try { @@ -89,6 +82,13 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId, } } + this.serviceType = serviceType; + this.appId = StringUtils.isEmpty(appId) ? serviceType : appId; + this.propertyPrefix = "ranger.plugin." + serviceType; + this.serviceName = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName; + + addResourcesForServiceName(this.serviceType, this.serviceName); + String trustedProxyAddressString = this.get(propertyPrefix + ".trusted.proxy.ipaddresses"); if (StringUtil.isEmpty(clusterName)) { diff --git a/agents-common/src/test/java/org/apache/ranger/authorization/hadoop/config/TestRangerPluginConfig.java b/agents-common/src/test/java/org/apache/ranger/authorization/hadoop/config/TestRangerPluginConfig.java index 296f6ff846a..ff8d538487b 100644 --- a/agents-common/src/test/java/org/apache/ranger/authorization/hadoop/config/TestRangerPluginConfig.java +++ b/agents-common/src/test/java/org/apache/ranger/authorization/hadoop/config/TestRangerPluginConfig.java @@ -79,7 +79,7 @@ public void test02_constructor_serviceNameFromConfig_whenNullInCtor() throws Exc RangerPluginConfig cfg = new RangerPluginConfig("hdfs", null, null, null, null, files, null); - assertNull(cfg.getServiceName()); + assertEquals("svcA", cfg.getServiceName()); assertEquals("svcA", cfg.get(key)); } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 2ae7d14e0f2..ec830a91a6e 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -58,13 +58,29 @@ public class AuditREST { private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); - private static final Map> allowedServiceUsers; + private static volatile Map> allowedServiceUsers; static { - allowedServiceUsers = initializeAllowedUsers(); initializeAuthToLocal(); } + private static Map> getAllowedServiceUsers() { + Map> ret = allowedServiceUsers; + + if (ret == null) { + synchronized (AuditREST.class) { + ret = allowedServiceUsers; + + if (ret == null) { + allowedServiceUsers = initializeAllowedUsers(); + ret = allowedServiceUsers; + } + } + } + + return ret; + } + @Autowired AuditDestinationMgr auditDestinationMgr; @@ -330,7 +346,7 @@ private boolean isAllowedServiceUser(String serviceName, String userName) { boolean ret; if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) { - Set allowedUsers = allowedServiceUsers.get(serviceName); + Set allowedUsers = getAllowedServiceUsers().get(serviceName); ret = allowedUsers != null && allowedUsers.contains(userName); } else { diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 1688b4dc43d..d46528c4984 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -242,6 +242,12 @@ Allowed users for dev_solr (Solr plugin) + + ranger.audit.ingestor.service.dev_elasticsearch.allowed.users + elasticsearch + Allowed users for dev_elasticsearch (Elasticsearch plugin) + + ranger.audit.ingestor.auth.to.local @@ -249,6 +255,7 @@ RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ + RULE:[2:$1/$2@$0](elasticsearch/.*@.*)s/.*/elasticsearch/ RULE:[1:$1@$0](.*@.*)s/@.*// DEFAULT diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore index 05026e91e9d..06204248b97 100644 --- a/dev-support/ranger-docker/.dockerignore +++ b/dev-support/ranger-docker/.dockerignore @@ -17,5 +17,6 @@ !dist/ranger-*-trino-plugin.tar.gz !dist/ranger-*-ozone-plugin.tar.gz !dist/ranger-*-solr-plugin.tar.gz +!dist/ranger-*-elasticsearch-plugin.tar.gz !downloads/* !scripts/* diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 9ec9bdf7c02..446d79a8af5 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -88,6 +88,13 @@ TRINO_VERSION=latest # Open Search OPENSEARCH_VERSION=1.3.19 +# Elasticsearch Configuration (authorization plugin testing) +ELASTICSEARCH_VERSION=7.17.29 +ELASTICSEARCH_PLUGIN_VERSION=3.0.0-SNAPSHOT +ELASTICSEARCH_BOOTSTRAP_PASSWORD=rangerR0cks! +RANGER_ADMIN_USER=admin +RANGER_ADMIN_PASSWORD=rangerR0cks! + # Debug Configuration DEBUG_ADMIN=false DEBUG_USERSYNC=false diff --git a/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch b/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch new file mode 100644 index 00000000000..da42a46a390 --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG RANGER_BASE_IMAGE=apache/ranger-base +ARG RANGER_BASE_VERSION=20260123-2-8 +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} + +ARG ELASTICSEARCH_VERSION +ARG ELASTICSEARCH_PLUGIN_VERSION + +COPY ./dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz /home/ranger/dist/ +COPY ./downloads/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz /home/ranger/dist/ +COPY ./scripts/elasticsearch/*.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/elasticsearch/patch-ranger-security-xml.py ${RANGER_SCRIPTS}/ +COPY ./scripts/elasticsearch/patch-ranger-audit-xml.py ${RANGER_SCRIPTS}/ +COPY ./downloads/jaxb-api-2.2.11.jar ${RANGER_SCRIPTS}/elasticsearch-lib/ +COPY ./downloads/jaxb-runtime-2.3.2.jar ${RANGER_SCRIPTS}/elasticsearch-lib/ +COPY ./downloads/javax.activation-api-1.2.0.jar ${RANGER_SCRIPTS}/elasticsearch-lib/ + +RUN groupadd -r elasticsearch 2>/dev/null || true && \ + useradd -r -g hadoop -d /opt/elasticsearch -s /bin/bash elasticsearch 2>/dev/null || true && \ + tar xvfz /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz --directory=/opt/ && \ + ln -s /opt/elasticsearch-${ELASTICSEARCH_VERSION} /opt/elasticsearch && \ + rm -f /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz && \ + mkdir -p /opt/elasticsearch/data /opt/elasticsearch/logs && \ + tar xvfz /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz --directory=/opt/ranger && \ + ln -s /opt/ranger/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin /opt/ranger/ranger-elasticsearch-plugin && \ + rm -f /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz && \ + rm -f /opt/ranger/ranger-elasticsearch-plugin/install.properties && \ + chown -R elasticsearch:hadoop /opt/elasticsearch* && \ + chmod 744 ${RANGER_SCRIPTS}/ranger-elasticsearch-setup.sh \ + ${RANGER_SCRIPTS}/ranger-elasticsearch-post-setup.sh \ + ${RANGER_SCRIPTS}/ranger-elasticsearch.sh + +ENV ELASTICSEARCH_HOME=/opt/elasticsearch +ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/elasticsearch/bin + +ENTRYPOINT [ "/home/ranger/scripts/ranger-elasticsearch.sh" ] diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index 76c9968a32d..bab29ecbeea 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -34,7 +34,7 @@ Use Dockerfiles in this directory to create docker images and run them to build ~~~ chmod +x download-archives.sh # use a subset of the below to download specific services - ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch + ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch elasticsearch ~~~ - Execute following commands to set environment variables to build Apache Ranger docker containers: @@ -157,6 +157,41 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml up docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-opensearch.yml up -d ~~~ +#### Bring up elasticsearch container (authorization plugin testing): +~~~ +# Prerequisites: build Ranger artifacts and download the Elasticsearch archive +mvn clean package -pl distro -am -DskipTests +cp target/ranger-* dev-support/ranger-docker/dist/ +cd dev-support/ranger-docker +./download-archives.sh elasticsearch + +export RANGER_DB_TYPE=postgres + +# Host port 9201 maps to container 9200 (avoids conflict with OpenSearch on 9200). +# On Linux, ensure vm.max_map_count >= 262144 (e.g. sudo sysctl -w vm.max_map_count=262144). +docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml \ + -f docker-compose.ranger-elasticsearch.yml up -d --build +~~~ + +Elasticsearch starts with X-Pack Security enabled (native realm). Default credentials: + +- `elastic` / value of `ELASTICSEARCH_BOOTSTRAP_PASSWORD` in `.env` (default: `rangerR0cks!`) +- `testuser_2` / same password (created automatically for authorization testing) + +Smoke tests after the container is healthy: + +~~~ +# Authenticated request (expect 200 or policy-based 403, not 401) +curl -u elastic:rangerR0cks! http://localhost:9201/test-index/_search + +# Unauthenticated request (expect 401) +curl http://localhost:9201/test-index/_search +~~~ + +Ranger Admin registers the `dev_elasticsearch` service automatically on first startup. +The plugin polls policies from Ranger Admin; allow up to 30 seconds after startup for +policy refresh before running authorization tests. + #### OpenSearch audit flow (replace Solr for access audits) OpenSearch can replace Solr for **audit storage and UI queries**. Ranger Admin reads audits via diff --git a/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml b/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml new file mode 100644 index 00000000000..3acf5fa9ec6 --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml @@ -0,0 +1,58 @@ +services: + ranger-elasticsearch: + build: + context: . + dockerfile: Dockerfile.ranger-elasticsearch + args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} + - ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION} + - ELASTICSEARCH_PLUGIN_VERSION=${ELASTICSEARCH_PLUGIN_VERSION} + image: ranger-elasticsearch + container_name: ranger-elasticsearch + hostname: ranger-elasticsearch.rangernw + volumes: + - ./dist/version:/home/ranger/dist/version:ro + - ./dist/keytabs/ranger-elasticsearch:/etc/keytabs + - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro + - ./scripts/elasticsearch/elasticsearch.yml:/home/ranger/scripts/elasticsearch.yml:ro + - ./scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties:/opt/ranger/ranger-elasticsearch-plugin/install.properties + - ./scripts/elasticsearch/patch-ranger-audit-xml.py:/home/ranger/scripts/patch-ranger-audit-xml.py:ro + - ./scripts/elasticsearch/ranger-elasticsearch.sh:/home/ranger/scripts/ranger-elasticsearch.sh:ro + - elasticsearch-data:/opt/elasticsearch/data + - elasticsearch-logs:/opt/elasticsearch/logs + stdin_open: true + tty: true + networks: + - ranger + ports: + - "9201:9200" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + depends_on: + ranger-kdc: + condition: service_started + ranger: + condition: service_started + environment: + - KERBEROS_ENABLED=${KERBEROS_ENABLED} + - KRB5_CONFIG=/etc/krb5.conf + - ELASTICSEARCH_VERSION + - ELASTICSEARCH_PLUGIN_VERSION + - ELASTICSEARCH_BOOTSTRAP_PASSWORD + - RANGER_ADMIN_USER + - RANGER_ADMIN_PASSWORD + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + +volumes: + elasticsearch-data: + elasticsearch-logs: + +networks: + ranger: + name: rangernw diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index b107cc50d29..06300756139 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -17,7 +17,7 @@ # limitations under the License. # -# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone archives to a local cache directory. +# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone/OpenSearch/Elasticsearch archives to a local cache directory. # The downloaded archives will be used while building docker images that run these services. # @@ -139,6 +139,7 @@ then downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz https://archive.apache.org/dist/knox/${KNOX_VERSION} extractOzoneIfNeeded downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION} + downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch else for arg in "$@"; do if [[ $arg == 'hadoop' ]] @@ -164,6 +165,12 @@ else elif [[ $arg == 'opensearch' ]] then downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION} + elif [[ $arg == 'elasticsearch' ]] + then + downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch + downloadIfNotPresent jaxb-api-2.2.11.jar https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.2.11 + downloadIfNotPresent jaxb-runtime-2.3.2.jar https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/2.3.2 + downloadIfNotPresent javax.activation-api-1.2.0.jar https://repo1.maven.org/maven2/javax/activation/javax.activation-api/1.2.0 else echo "Passed argument $arg is invalid!" fi diff --git a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py index 7e2eae0ee0f..1dc95c948a3 100644 --- a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py +++ b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py @@ -148,11 +148,24 @@ def service_not_exists(service): 'ranger.plugin.super.users': 'solr', 'ranger.plugin.solr.policy.refresh.synchronous':'true'}}) -services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone, solr] +elasticsearch = RangerService({'name': 'dev_elasticsearch', 'type': 'elasticsearch', + 'configs': {'username': 'elastic', + 'elasticsearch.url': 'http://ranger-elasticsearch.rangernw:9200', + 'policy.download.auth.users': 'elastic,admin,elasticsearch', + 'tag.download.auth.users': 'elastic,admin,elasticsearch', + 'userstore.download.auth.users': 'elastic,admin,elasticsearch', + 'setup.additional.default.policies': 'true', + 'default-policy.1.name': 'index: test-index', + 'default-policy.1.resource.index': 'test-index', + 'default-policy.1.policyItem.1.users': 'elastic,testuser_2', + 'default-policy.1.policyItem.1.accessTypes': 'read,all', + 'ranger.plugin.elasticsearch.policy.refresh.synchronous': 'true'}}) + +services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone, solr, elasticsearch] for service in services: try: if service_not_exists(service): ranger_client.create_service(service) print(f" {service.name} service created!") except Exception as e: - print(f"An exception occured: {e}") + print(f"An exception occured: {e}") \ No newline at end of file diff --git a/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml b/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml new file mode 100644 index 00000000000..826f5375e90 --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster.name: ranger-elasticsearch-cluster +node.name: ranger-elasticsearch.rangernw +network.host: 0.0.0.0 +discovery.type: single-node + +path.data: /opt/elasticsearch/data +path.logs: /opt/elasticsearch/logs + +xpack.security.enabled: true +xpack.security.http.ssl.enabled: false +xpack.security.transport.ssl.enabled: false +xpack.ml.enabled: false diff --git a/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py new file mode 100644 index 00000000000..bafad172b7e --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import re + + +def set_property(text, name, value): + pattern = rf"({re.escape(name)}\s*)[^<]*()" + if re.search(pattern, text): + return re.sub(pattern, lambda match: f"{match.group(1)}{value}{match.group(2)}", text, count=1) + + insertion = ( + f"\t\n" + f"\t\t{name}\n" + f"\t\t{value}\n" + f"\t\n" + f"" + ) + return text.replace("", insertion, 1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("audit_xml") + parser.add_argument("--auditserver-url") + parser.add_argument("--spool-dir") + parser.add_argument("--kerberos-enabled", action="store_true") + args = parser.parse_args() + + with open(args.audit_xml, encoding="utf-8") as handle: + text = handle.read() + + text = set_property(text, "xasecure.audit.is.enabled", "true") + text = set_property(text, "xasecure.audit.destination.auditserver", "true") + + if args.auditserver_url: + text = set_property(text, "xasecure.audit.destination.auditserver.url", args.auditserver_url) + + if args.spool_dir: + text = set_property( + text, + "xasecure.audit.destination.auditserver.batch.filespool.dir", + args.spool_dir, + ) + + if args.kerberos_enabled: + text = set_property(text, "xasecure.audit.destination.auditserver.authn.type", "kerberos") + + with open(args.audit_xml, "w", encoding="utf-8") as handle: + handle.write(text) + + +if __name__ == "__main__": + main() diff --git a/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py new file mode 100644 index 00000000000..dfd85e1f530 --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import re +import sys + + +def set_property(text, name, value): + pattern = rf"({re.escape(name)}\s*)[^<]*()" + if re.search(pattern, text): + return re.sub(pattern, lambda match: f"{match.group(1)}{value}{match.group(2)}", text, count=1) + + insertion = ( + f"\t\n" + f"\t\t{name}\n" + f"\t\t{value}\n" + f"\t\n" + f"" + ) + return text.replace("", insertion, 1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("security_xml") + parser.add_argument("--cache-dir") + parser.add_argument("--poll-interval-ms") + parser.add_argument("--admin-user") + parser.add_argument("--admin-password") + parser.add_argument("--clear-admin-creds", action="store_true") + args = parser.parse_args() + + with open(args.security_xml, encoding="utf-8") as handle: + text = handle.read() + + if args.cache_dir: + text = set_property(text, "ranger.plugin.elasticsearch.policy.cache.dir", args.cache_dir) + + if args.poll_interval_ms: + text = set_property(text, "ranger.plugin.elasticsearch.policy.pollIntervalMs", args.poll_interval_ms) + + if args.clear_admin_creds: + text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.username", "") + text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.password", "") + else: + if args.admin_user is not None: + text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.username", args.admin_user) + if args.admin_password is not None: + text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.password", args.admin_password) + + with open(args.security_xml, "w", encoding="utf-8") as handle: + handle.write(text) + + +if __name__ == "__main__": + main() diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties new file mode 100644 index 00000000000..82e94f7c6b6 --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +POLICY_MGR_URL=http://ranger.rangernw:6080 +REPOSITORY_NAME=dev_elasticsearch +COMPONENT_INSTALL_DIR_NAME=/opt/elasticsearch +POLICY_CACHE_FILE_PATH=/opt/elasticsearch/data/ranger-policycache +RANGER_ADMIN_USER=admin +RANGER_ADMIN_PASSWORD=rangerR0cks! + +CUSTOM_USER=elasticsearch +CUSTOM_GROUP=hadoop + +XAAUDIT.SUMMARY.ENABLE=true + +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr.rangernw:8983/solr/ranger_audits + +# Following properties are needed to get past installation script! Please don't remove +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit +XAAUDIT.HDFS.DESTINTATION_FILE=hadoop +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/elasticsearch/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/elasticsearch/audit/archive +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +XAAUDIT.SOLR.ENABLE=false +XAAUDIT.SOLR.URL=NONE +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/elasticsearch/audit/solr/spool +XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=false + +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=com.sun.security.auth.module.Krb5LoginModule +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=required +XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=true +XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=true +XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=true +XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=elasticsearch +XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=/etc/keytabs/elasticsearch.keytab +XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=elasticsearch/ranger-elasticsearch.rangernw@EXAMPLE.COM + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.HDFS.ENABLE=false +XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop.rangernw:9000/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/elasticsearch/audit/hdfs/spool + +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +XAAUDIT.LOG4J.ENABLE=false +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=false +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +XAAUDIT.AUDITSERVER.ENABLE=true +XAAUDIT.AUDITSERVER.URL=http://ranger-audit-ingestor.rangernw:7081 +XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/opt/elasticsearch/data/ranger-audit-spool +XAAUDIT.AUDITSERVER.AUTHN.TYPE=kerberos + +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh new file mode 100755 index 00000000000..2631014b93f --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +ES_URL="http://127.0.0.1:9200" +ES_USER="elastic" +ES_PASS="${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}" +RANGER_URL="http://ranger.rangernw:6080" +RANGER_USER="${RANGER_ADMIN_USER:-admin}" +RANGER_PASS="${RANGER_ADMIN_PASSWORD:-rangerR0cks!}" +SERVICE_NAME="dev_elasticsearch" +CACHE_DIR="${ELASTICSEARCH_HOME}/data/ranger-policycache" +CACHE_FILE="${CACHE_DIR}/elasticsearch_${SERVICE_NAME}.json" +MAX_ATTEMPTS=60 +MODE="${1:-all}" + +wait_for_ranger_admin() { + local attempt=0 + until curl -s --max-time 5 -u "${RANGER_USER}:${RANGER_PASS}" \ + "${RANGER_URL}/service/public/v2/api/version" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "${attempt}" -ge "${MAX_ATTEMPTS}" ]; then + echo "ERROR: Ranger Admin did not become reachable in time." >&2 + exit 1 + fi + sleep 5 + done +} + +wait_for_authorized_search() { + local attempt=0 + local http_code="" + until http_code="$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -u "${ES_USER}:${ES_PASS}" \ + "${ES_URL}/test-index/_search")" && echo "${http_code}" | grep -qE '^(200|404)$'; do + attempt=$((attempt + 1)) + if [ "${attempt}" -ge "${MAX_ATTEMPTS}" ]; then + echo "ERROR: Elasticsearch did not authorize ${ES_USER} in time." >&2 + exit 1 + fi + sleep 5 + done +} + +seed_policy_cache() { + echo "Seeding Ranger policy cache under ${CACHE_DIR}..." + mkdir -p "${CACHE_DIR}" + python3 - "${RANGER_URL}" "${RANGER_USER}" "${RANGER_PASS}" "${SERVICE_NAME}" "${CACHE_FILE}" <<'PY' +import json +import sys +import urllib.request +import base64 + +ranger_url, user, password, service_name, cache_file = sys.argv[1:6] +auth = base64.b64encode(f"{user}:{password}".encode()).decode() + +def fetch(path): + req = urllib.request.Request(f"{ranger_url}{path}") + req.add_header("Authorization", f"Basic {auth}") + with urllib.request.urlopen(req) as resp: + return json.load(resp) + +service = fetch(f"/service/public/v2/api/service/name/{service_name}") +policies = fetch(f"/service/public/v2/api/service/{service_name}/policy") +if not isinstance(policies, list): + raise SystemExit(f"unexpected policy payload type: {type(policies)}") + +for policy in policies: + resources = policy.get("resources") or {} + index_resource = resources.get("index") or {} + index_values = index_resource.get("values") or [] + if "test-index" in index_values: + for item in policy.get("policyItems") or []: + item["users"] = ["elastic", "testuser_2"] + policy_id = policy.get("id") + if policy_id is not None: + update_body = { + "id": policy_id, + "name": policy.get("name"), + "service": policy.get("service", service_name), + "serviceType": policy.get("serviceType", "elasticsearch"), + "resources": policy.get("resources"), + "policyItems": policy.get("policyItems"), + "isEnabled": policy.get("isEnabled", True), + "policyType": policy.get("policyType", 0), + "isAuditEnabled": policy.get("isAuditEnabled", True), + } + try: + update_req = urllib.request.Request( + f"{ranger_url}/service/public/v2/api/policy/{policy_id}", + data=json.dumps(update_body).encode(), + method="PUT", + ) + update_req.add_header("Authorization", f"Basic {auth}") + update_req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(update_req) as resp: + if resp.status != 200: + print(f"WARN: Ranger Admin policy update returned HTTP {resp.status}", file=sys.stderr) + except Exception as exc: + print(f"WARN: Could not update test-index policy in Ranger Admin: {exc}", file=sys.stderr) + +policy_version = service.get("policyVersion") +if policy_version is None: + policy_version = max((p.get("version") or 1) for p in policies) if policies else 1 + +payload = { + "serviceName": service_name, + "serviceId": service.get("id", 1), + "policyVersion": policy_version, + "policies": policies, +} + +with open(cache_file, "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY + + chown elasticsearch:hadoop "${CACHE_FILE}" + chmod 640 "${CACHE_FILE}" +} + +disable_admin_policy_download() { + SECURITY_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-security.xml" + if [ -f "${SECURITY_XML}" ]; then + python3 "${RANGER_SCRIPTS}/patch-ranger-security-xml.py" "${SECURITY_XML}" --clear-admin-creds + fi +} + +create_test_fixtures() { + echo "Creating test index test-index..." + curl -s --max-time 30 -u "${ES_USER}:${ES_PASS}" -X PUT "${ES_URL}/test-index" \ + -H 'Content-Type: application/json' \ + -d '{"settings":{"number_of_shards":1,"number_of_replicas":0}}' \ + | grep -q '"acknowledged":true' || echo "test-index may already exist" +} + +case "${MODE}" in + seed) + wait_for_ranger_admin + seed_policy_cache + ;; + fixtures) + sleep 30 + wait_for_authorized_search + create_test_fixtures + ;; + all) + wait_for_ranger_admin + seed_policy_cache + sleep 30 + wait_for_authorized_search + create_test_fixtures + ;; + *) + echo "Usage: $0 [seed|fixtures|all]" >&2 + exit 1 + ;; +esac + +echo "Elasticsearch post-setup (${MODE}) completed successfully" diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh new file mode 100755 index 00000000000..91fd1660d3e --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +cp ${RANGER_SCRIPTS}/elasticsearch.yml ${ELASTICSEARCH_HOME}/config/elasticsearch.yml + +if [ ! -f "${ELASTICSEARCH_HOME}/config/elasticsearch.keystore" ]; then + su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-keystore create" + echo "${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}" | \ + su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-keystore add -x bootstrap.password" +fi + +chown -R elasticsearch:hadoop ${ELASTICSEARCH_HOME} + +# File-realm users for authorization smoke tests (offline; Ranger blocks security REST APIs). +ES_BOOTSTRAP_PASSWORD="${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}" +cat > "${ELASTICSEARCH_HOME}/config/roles.yml" <<'EOF' +ranger_test_index_reader: + indices: + - names: ['test-index'] + privileges: ['read', 'view_index_metadata'] +EOF +chown elasticsearch:hadoop "${ELASTICSEARCH_HOME}/config/roles.yml" +chmod 640 "${ELASTICSEARCH_HOME}/config/roles.yml" +for smoke_user in testuser_2 testuser_denied; do + su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-users useradd ${smoke_user} -p \"${ES_BOOTSTRAP_PASSWORD}\" -r ranger_test_index_reader" \ + 2>/dev/null || true +done + +# ES plugin security manager allows read but not write under /etc/ranger; keep cache under ES data. +export POLICY_CACHE_FILE_PATH="${ELASTICSEARCH_HOME}/data/ranger-policycache" +mkdir -p "${POLICY_CACHE_FILE_PATH}" +chown elasticsearch:hadoop "${POLICY_CACHE_FILE_PATH}" +chmod 750 "${POLICY_CACHE_FILE_PATH}" + +AUDIT_SPOOL_DIR="${ELASTICSEARCH_HOME}/data/ranger-audit-spool" +mkdir -p "${AUDIT_SPOOL_DIR}" +chown elasticsearch:hadoop "${AUDIT_SPOOL_DIR}" +chmod 750 "${AUDIT_SPOOL_DIR}" + +cd ${RANGER_HOME}/ranger-elasticsearch-plugin +./enable-elasticsearch-plugin.sh + +# enable-agent.sh resets POLICY_CACHE_FILE_PATH to /etc/ranger/...; patch Ranger config for docker. +SECURITY_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-security.xml" +if [ -f "${SECURITY_XML}" ]; then + python3 "${RANGER_SCRIPTS}/patch-ranger-security-xml.py" "${SECURITY_XML}" \ + --cache-dir "${POLICY_CACHE_FILE_PATH}" \ + --poll-interval-ms "86400000" \ + --admin-user "${RANGER_ADMIN_USER:-admin}" \ + --admin-password "${RANGER_ADMIN_PASSWORD:-rangerR0cks!}" +fi + +AUDIT_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-audit.xml" +if [ -f "${AUDIT_XML}" ]; then + python3 "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" "${AUDIT_XML}" \ + --auditserver-url "http://ranger-audit-ingestor.rangernw:7081" \ + --spool-dir "${AUDIT_SPOOL_DIR}" +fi + +# Replace plugin symlinks with local copies; ES security manager cannot read jars via /opt/ranger symlinks. +RANGER_PLUGIN_LIB="${RANGER_HOME}/ranger-elasticsearch-plugin/lib/ranger-elasticsearch-plugin" +PLUGIN_DIR="${ELASTICSEARCH_HOME}/plugins/ranger-elasticsearch-plugin" +rm -rf "${PLUGIN_DIR}" +mkdir -p "${PLUGIN_DIR}" +cp -a "${RANGER_PLUGIN_LIB}/." "${PLUGIN_DIR}/" +# ES 7.17+ does not allow plugins to create classloaders; load impl jars on the plugin classpath. +if [ -d "${RANGER_PLUGIN_LIB}/ranger-elasticsearch-plugin-impl" ]; then + cp "${RANGER_PLUGIN_LIB}/ranger-elasticsearch-plugin-impl/"*.jar "${PLUGIN_DIR}/" +fi +# Shim runtime deps (exclude jars already provided by x-pack-security / x-pack-core / ES lib) +for jar in commons-lang3 commons-collections hadoop-client-api hadoop-client-runtime commons-configuration gson jackson-databind jackson-annotations; do + cp "${RANGER_HOME}/ranger-elasticsearch-plugin/install/lib/${jar}-"*.jar "${PLUGIN_DIR}/" 2>/dev/null || true +done +if [ -d "${RANGER_SCRIPTS}/elasticsearch-lib" ]; then + cp "${RANGER_SCRIPTS}/elasticsearch-lib/"*.jar "${PLUGIN_DIR}/" 2>/dev/null || true +fi +cp "${RANGER_HOME}/ranger-elasticsearch-plugin/lib/ranger-elasticsearch-plugin/plugin-security.policy" "${PLUGIN_DIR}/" 2>/dev/null || true +chown -R elasticsearch:hadoop "${PLUGIN_DIR}" "${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin" + +echo "Elasticsearch Ranger plugin setup completed successfully" diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh new file mode 100755 index 00000000000..9551dd5c6ad --- /dev/null +++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -e ${ELASTICSEARCH_HOME}/.setupDone ] +then + if "${RANGER_SCRIPTS}"/ranger-elasticsearch-setup.sh; + then + touch "${ELASTICSEARCH_HOME}"/.setupDone + else + echo "Ranger Elasticsearch Setup Script didn't complete proper execution." >&2 + exit 1 + fi +fi + +AUDIT_SPOOL_DIR="${ELASTICSEARCH_HOME}/data/ranger-audit-spool" +mkdir -p "${AUDIT_SPOOL_DIR}" +chown elasticsearch:hadoop "${AUDIT_SPOOL_DIR}" 2>/dev/null || true +chmod 750 "${AUDIT_SPOOL_DIR}" 2>/dev/null || true + +AUDIT_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-audit.xml" +if [ -f "${AUDIT_XML}" ] && [ -f "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" ]; then + KERBEROS_PATCH_ARG="" + if [ "${KERBEROS_ENABLED}" = "true" ]; then + KERBEROS_PATCH_ARG="--kerberos-enabled" + fi + python3 "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" "${AUDIT_XML}" \ + --auditserver-url "http://ranger-audit-ingestor.rangernw:7081" \ + --spool-dir "${AUDIT_SPOOL_DIR}" \ + ${KERBEROS_PATCH_ARG} + chown elasticsearch:hadoop "${AUDIT_XML}" 2>/dev/null || true +fi + +if [ ! -e ${ELASTICSEARCH_HOME}/.postSetupDone ] +then + if ! "${RANGER_SCRIPTS}"/ranger-elasticsearch-post-setup.sh seed; + then + echo "Ranger Elasticsearch Post-Setup Script didn't complete proper execution." >&2 + exit 1 + fi +fi + +su -s /bin/bash elasticsearch -c "cd ${ELASTICSEARCH_HOME} && ES_JAVA_OPTS='${ES_JAVA_OPTS}' ./bin/elasticsearch" & +ES_PID=$! + +if [ ! -e ${ELASTICSEARCH_HOME}/.postSetupDone ] +then + if "${RANGER_SCRIPTS}"/ranger-elasticsearch-post-setup.sh fixtures; + then + touch "${ELASTICSEARCH_HOME}"/.postSetupDone + else + echo "Ranger Elasticsearch Post-Setup Script didn't complete proper execution." >&2 + kill "${ES_PID}" 2>/dev/null || true + exit 1 + fi +fi + +if ! ps -p "${ES_PID}" > /dev/null 2>&1 +then + echo "The Elasticsearch process exited unexpectedly." >&2 + exit 1 +fi + +tail --pid="${ES_PID}" -f /dev/null diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh index 067042ada8c..ee212d49e5f 100755 --- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh +++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh @@ -62,11 +62,7 @@ if [ "${KERBEROS_ENABLED}" == "true" ]; then # Kerberos service name sasl.kerberos.service.name=kafka - # Ranger authorization - authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer - - # Super users bypass Ranger authorization for admin operations - super.users=User:kafka + # Broker-only for audit pipeline; Ranger Kafka authorizer needs audit-core on broker classpath. EOF else echo "Configuring Kafka with PLAINTEXT (no Kerberos)" diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh index 4946cc0262d..39f80015882 100644 --- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh +++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh @@ -121,7 +121,9 @@ function create_keytabs() { create_principal_and_keytab opensearch ranger-opensearch create_principal_and_keytab HTTP ranger-opensearch - + + create_principal_and_keytab elasticsearch ranger-elasticsearch + create_principal_and_keytab om om create_principal_and_keytab scm scm create_principal_and_keytab dn datanode diff --git a/distro/pom.xml b/distro/pom.xml index fa97c3fbd12..1a16ee94003 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -29,6 +29,30 @@ Apache Ranger Distribution + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.databind.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + provided + org.apache.ranger audit-dispatcher-hdfs diff --git a/distro/src/main/assembly/pdp.xml b/distro/src/main/assembly/pdp.xml index 853248416ed..8d355b5d627 100644 --- a/distro/src/main/assembly/pdp.xml +++ b/distro/src/main/assembly/pdp.xml @@ -157,6 +157,19 @@ + + + + false + lib + provided + + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + + + + diff --git a/distro/src/main/assembly/plugin-elasticsearch.xml b/distro/src/main/assembly/plugin-elasticsearch.xml index 777437f24dd..0471da5bfd6 100644 --- a/distro/src/main/assembly/plugin-elasticsearch.xml +++ b/distro/src/main/assembly/plugin-elasticsearch.xml @@ -37,8 +37,9 @@ 644 commons-collections:commons-collections - org.slf4j:slf4j-api - org.slf4j:slf4j-log4j12 + org.apache.commons:commons-lang3 + org.apache.httpcomponents:httpclient + org.apache.httpcomponents:httpcore @@ -56,65 +57,40 @@ org.apache.ranger:ranger-elasticsearch-plugin - lib/ranger-elasticsearch-plugin/ranger-elasticsearch-plugin-impl - false + lib/ranger-elasticsearch-plugin + true false 755 644 - com.carrotsearch:hppc - com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version} - com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version} - com.fasterxml.jackson.core:jackson-databind:${fasterxml.jackson.version} - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version} - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version} + org.apache.ranger:ranger-audit-core + org.apache.ranger:ranger-audit-dest-auditserver + org.apache.ranger:ranger-authz-api + org.apache.ranger:ranger-common-utils + org.apache.ranger:ranger-plugins-cred + org.apache.ranger:ranger-plugins-common + org.apache.ranger:ugsync-util + org.apache.ranger:ranger-elasticsearch-plugin com.google.code.gson:gson - commons-codec:commons-codec - commons-collections:commons-collections - commons-configuration:commons-configuration:jar:${commons.configuration.version} - commons-io:commons-io - commons-logging:commons-logging:jar:${commons.logging.version} - io.airlift:aircompressor:jar:${aircompressor.version} - joda-time:joda-time - org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + commons-configuration:commons-configuration + javax.ws.rs:javax.ws.rs-api + javax.annotation:javax.annotation-api + org.glassfish.jersey.core:jersey-client + org.glassfish.jersey.core:jersey-common + org.glassfish.jersey.inject:jersey-hk2 + org.glassfish.jersey.ext:jersey-entity-filtering + org.glassfish.jersey.media:jersey-media-json-jackson + org.glassfish.hk2:hk2-api + org.glassfish.hk2:hk2-locator + org.glassfish.hk2:hk2-utils + org.glassfish.hk2.external:aopalliance-repackaged + org.glassfish.hk2.external:jakarta.inject + org.javassist:javassist + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider + com.fasterxml.jackson.module:jackson-module-jaxb-annotations org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} - org.apache.hive:hive-storage-api:jar:${hive.storage-api.version} - org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version} - org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version} - org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version} - org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version} - org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version} - org.apache.lucene:lucene-core - org.apache.orc:orc-core:jar:${orc.version} - org.apache.orc:orc-shims:jar:${orc.version} - org.apache.solr:solr-solrj:jar:${solr.version} - org.eclipse.jetty:jetty-client:jar:${jetty-client.version} - org.elasticsearch.client:elasticsearch-rest-client - org.elasticsearch.client:elasticsearch-rest-high-level-client - org.elasticsearch.plugin:lang-mustache-client - org.elasticsearch.plugin:rank-eval-client - org.elasticsearch:elasticsearch-core - org.elasticsearch:elasticsearch-x-content - org.elasticsearch:elasticsearch - org.noggit:noggit:jar:${noggit.version} - - org.graalvm.js:js-language:jar:${graalvm.version} - org.graalvm.js:js-scriptengine:jar:${graalvm.version} - org.graalvm.polyglot:polyglot:jar:${graalvm.version} - org.graalvm.regex:regex:jar:${graalvm.version} - org.graalvm.truffle:truffle-api:jar:${graalvm.version} - org.graalvm.truffle:truffle-runtime:jar:${graalvm.version} - org.graalvm.truffle:truffle-compiler:jar:${graalvm.version} - org.graalvm.sdk:jniutils:jar:${graalvm.version} - org.graalvm.sdk:collections:jar:${graalvm.version} - org.graalvm.sdk:nativeimage:jar:${graalvm.version} - org.graalvm.sdk:word:jar:${graalvm.version} - org.graalvm.shadowed:icu4j:jar:${graalvm.version} - org.graalvm.shadowed:xz:jar:${graalvm.version} - @@ -127,7 +103,7 @@ install/lib - false + true false 755 644 @@ -218,5 +194,33 @@ /lib/ranger-elasticsearch-plugin 755 + + + ${settings.localRepository}/com/fasterxml/jackson/core/jackson-databind/${elasticsearch.jackson.version}/jackson-databind-${elasticsearch.jackson.version}.jar + lib/ranger-elasticsearch-plugin + + + ${settings.localRepository}/com/fasterxml/jackson/core/jackson-annotations/${elasticsearch.jackson.version}/jackson-annotations-${elasticsearch.jackson.version}.jar + lib/ranger-elasticsearch-plugin + + + + false + lib/ranger-elasticsearch-plugin + provided + + javax.annotation:javax.annotation-api + + + + false + lib/ranger-elasticsearch-plugin + + javax.xml.bind:jaxb-api:jar:2.2.11 + org.glassfish.jaxb:jaxb-runtime:jar:2.3.2 + javax.activation:javax.activation-api:jar:1.2.0 + + + diff --git a/distro/src/main/assembly/plugin-solr.xml b/distro/src/main/assembly/plugin-solr.xml index 6398f3aeeda..206cebc2f8d 100644 --- a/distro/src/main/assembly/plugin-solr.xml +++ b/distro/src/main/assembly/plugin-solr.xml @@ -52,6 +52,14 @@ 755 644 + org.apache.ranger:ranger-audit-core + org.apache.ranger:ranger-audit-dest-auditserver + org.apache.ranger:ranger-authz-api + org.apache.ranger:ranger-common-utils + org.apache.ranger:ranger-plugins-cred + org.apache.ranger:ranger-plugins-common + org.apache.ranger:ugsync-util + org.apache.ranger:ranger-solr-plugin com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version} com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version} com.google.code.gson:gson diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg index c81a52746ff..0c9466481cd 100644 --- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg +++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg @@ -94,3 +94,4 @@ xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATIO xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.auditserver.authn.type %XAAUDIT.AUDITSERVER.AUTHN.TYPE% mod create-if-not-exists diff --git a/plugin-elasticsearch/pom.xml b/plugin-elasticsearch/pom.xml index 5f90340f179..b2b2cf85a45 100644 --- a/plugin-elasticsearch/pom.xml +++ b/plugin-elasticsearch/pom.xml @@ -31,6 +31,23 @@ UTF-8 + + + com.fasterxml.jackson.core + jackson-annotations + ${elasticsearch.jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${elasticsearch.jackson.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.google.protobuf protobuf-java diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties index 35ee159ae34..225486c4fc5 100644 --- a/plugin-elasticsearch/scripts/install.properties +++ b/plugin-elasticsearch/scripts/install.properties @@ -120,6 +120,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE XAAUDIT.AUDITSERVER.ENABLE=false XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081 XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool +XAAUDIT.AUDITSERVER.AUTHN.TYPE=%EMPTY% # End of V3 properties # diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java new file mode 100644 index 00000000000..964ef01a910 --- /dev/null +++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.authorization.elasticsearch.authorizer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.http.HttpStatus; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.elasticsearch.SpecialPermission; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.Base64; +import java.util.Collections; +import java.util.List; + +/** + * Posts access audits to the Ranger audit ingestor on the Elasticsearch request thread. + * Uses {@link HttpURLConnection} inside {@code SpecialPermission.check()} / {@code doPrivileged} + * so ES Security Manager allows outbound sockets (Jersey/async clients do not). + * When JAAS Kerberos settings are present in the audit configuration, posts use SPNEGO + * without {@code AuthenticatedURL} (ES plugin policy cannot grant cookie-handler permissions). + */ +final class ElasticsearchAuditIngestorClient { + private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchAuditIngestorClient.class); + + private static final String REST_PATH_POST = "/api/audit/access"; + private static final String QUERY_PARAM_SERVICE = "serviceName"; + private static final String QUERY_PARAM_APP_ID = "appId"; + private static final String AUTHZ_HEADER = "Authorization"; + private static final String NEGOTIATE_PREFIX = "Negotiate "; + private static final String WWW_AUTH_NEGOTIATE = "Negotiate"; + + private static final String JAAS_PRINCIPAL_PROP = "xasecure.audit.jaas.Client.option.principal"; + private static final String JAAS_KEYTAB_PROP = "xasecure.audit.jaas.Client.option.keyTab"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Object KERBEROS_INIT_LOCK = new Object(); + + private static volatile boolean kerberosInitialized; + + private ElasticsearchAuditIngestorClient() { + } + + static void init(Configuration config) { + if (config == null || kerberosInitialized || !isKerberosConfigured(config)) { + return; + } + + synchronized (KERBEROS_INIT_LOCK) { + if (kerberosInitialized || !isKerberosConfigured(config)) { + return; + } + + try { + SpecialPermission.check(); + + AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + initKerberos(config); + + return null; + }); + + kerberosInitialized = true; + } catch (PrivilegedActionException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + + LOG.error("Failed to initialize Kerberos for audit ingestor client: {}", cause.getMessage(), cause); + } catch (RuntimeException e) { + LOG.error("Failed to initialize Kerberos for audit ingestor client: {}", e.getMessage(), e); + } + } + } + + static boolean post(String auditServerBaseUrl, AuthzAuditEvent auditEvent) { + if (StringUtils.isBlank(auditServerBaseUrl) || auditEvent == null) { + return false; + } + + try { + SpecialPermission.check(); + + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> doPost(auditServerBaseUrl, auditEvent)); + } catch (PrivilegedActionException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + + LOG.error("Failed to post audit event to ingestor at {}: {}", auditServerBaseUrl, cause.getMessage(), cause); + + return false; + } catch (RuntimeException e) { + LOG.error("Failed to post audit event to ingestor at {}: {}", auditServerBaseUrl, e.getMessage(), e); + + return false; + } + } + + private static void initKerberos(Configuration config) throws IOException { + String principal = config.get(JAAS_PRINCIPAL_PROP); + String keytab = config.get(JAAS_KEYTAB_PROP); + + Configuration hadoopConf = new Configuration(false); + + hadoopConf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(hadoopConf); + System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); + UserGroupInformation.loginUserFromKeytab(principal, keytab); + + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + + LOG.info("Kerberos initialized for audit ingestor client. Principal: {}", loginUser != null ? loginUser.getUserName() : principal); + } + + private static boolean isKerberosConfigured(Configuration config) { + String principal = config.get(JAAS_PRINCIPAL_PROP); + String keytab = config.get(JAAS_KEYTAB_PROP); + + return StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(keytab) && !"%EMPTY%".equalsIgnoreCase(principal); + } + + private static boolean doPost(String auditServerBaseUrl, AuthzAuditEvent auditEvent) throws Exception { + if (kerberosInitialized && UserGroupInformation.isSecurityEnabled()) { + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + + if (loginUser != null && loginUser.hasKerberosCredentials()) { + return loginUser.doAs((PrivilegedExceptionAction) () -> doPostConnection(auditServerBaseUrl, auditEvent)); + } + } + + return doPostConnection(auditServerBaseUrl, auditEvent); + } + + private static boolean doPostConnection(String auditServerBaseUrl, AuthzAuditEvent auditEvent) throws Exception { + String baseUrl = auditServerBaseUrl.endsWith("/") ? auditServerBaseUrl.substring(0, auditServerBaseUrl.length() - 1) : auditServerBaseUrl; + String service = auditEvent.getRepositoryName(); + String appId = auditEvent.getAgentId(); + StringBuilder urlBuilder = new StringBuilder(baseUrl) + .append(REST_PATH_POST) + .append('?') + .append(QUERY_PARAM_SERVICE) + .append('=') + .append(URLEncoder.encode(service, StandardCharsets.UTF_8.name())); + + if (StringUtils.isNotBlank(appId)) { + urlBuilder.append('&') + .append(QUERY_PARAM_APP_ID) + .append('=') + .append(URLEncoder.encode(appId, StandardCharsets.UTF_8.name())); + } + + List payload = Collections.singletonList(auditEvent); + byte[] body = OBJECT_MAPPER.writeValueAsBytes(payload); + String url = urlBuilder.toString(); + URL requestUrl = new URL(url); + + if (kerberosInitialized) { + return doPostWithSpnego(requestUrl, body, auditEvent, service); + } + + return postOnce(requestUrl, body, null, auditEvent, service); + } + + private static boolean doPostWithSpnego(URL requestUrl, byte[] body, AuthzAuditEvent auditEvent, String service) throws Exception { + GSSContext context = createSpnegoContext(requestUrl.getHost()); + byte[] token = context.initSecContext(new byte[0], 0, 0); + String negotiateToken = token == null || token.length == 0 ? null : Base64.getEncoder().encodeToString(token); + + HttpPostResult result = postOnceWithStatus(requestUrl, body, negotiateToken); + + if (result.status == HttpStatus.SC_OK) { + logSuccess(service, auditEvent); + + return true; + } + + if (result.status == HttpStatus.SC_UNAUTHORIZED && result.wwwAuthenticate != null) { + byte[] challengeToken = extractNegotiateChallenge(result.wwwAuthenticate); + + if (challengeToken != null && challengeToken.length > 0) { + token = context.initSecContext(challengeToken, 0, challengeToken.length); + + if (token != null && token.length > 0) { + negotiateToken = Base64.getEncoder().encodeToString(token); + result = postOnceWithStatus(requestUrl, body, negotiateToken); + + if (result.status == HttpStatus.SC_OK) { + logSuccess(service, auditEvent); + + return true; + } + } + } + } + + LOG.error("Failed to post audit event to ingestor. HTTP status: {}", result.status); + + return false; + } + + private static boolean postOnce(URL requestUrl, byte[] body, String negotiateToken, AuthzAuditEvent auditEvent, String service) throws IOException { + HttpPostResult result = postOnceWithStatus(requestUrl, body, negotiateToken); + + if (result.status == HttpStatus.SC_OK) { + logSuccess(service, auditEvent); + + return true; + } + + LOG.error("Failed to post audit event to ingestor. HTTP status: {}", result.status); + + return false; + } + + private static HttpPostResult postOnceWithStatus(URL requestUrl, byte[] body, String negotiateToken) throws IOException { + HttpURLConnection connection = openConnection(requestUrl, body, negotiateToken); + int status = connection.getResponseCode(); + String wwwAuth = connection.getHeaderField("WWW-Authenticate"); + + drainResponse(connection, status); + + return new HttpPostResult(status, wwwAuth); + } + + private static HttpURLConnection openConnection(URL requestUrl, byte[] body, String negotiateToken) throws IOException { + HttpURLConnection connection = (HttpURLConnection) requestUrl.openConnection(); + + connection.setConnectTimeout(30_000); + connection.setReadTimeout(30_000); + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + + if (StringUtils.isNotBlank(negotiateToken)) { + connection.setRequestProperty(AUTHZ_HEADER, NEGOTIATE_PREFIX + negotiateToken); + } + + try (OutputStream outputStream = connection.getOutputStream()) { + outputStream.write(body); + } + + return connection; + } + + private static void drainResponse(HttpURLConnection connection, int status) { + try (InputStream stream = status >= HttpStatus.SC_BAD_REQUEST ? connection.getErrorStream() : connection.getInputStream()) { + if (stream != null) { + while (stream.read() >= 0) { + // drain so the connection can be reused or closed cleanly + } + } + } catch (IOException e) { + LOG.debug("Failed to drain audit ingestor response stream: {}", e.getMessage()); + } + } + + private static byte[] extractNegotiateChallenge(String wwwAuthenticateHeader) { + for (String headerValue : wwwAuthenticateHeader.split(",")) { + String trimmed = headerValue.trim(); + + if (trimmed.regionMatches(true, 0, WWW_AUTH_NEGOTIATE, 0, WWW_AUTH_NEGOTIATE.length())) { + String tokenPart = trimmed.substring(WWW_AUTH_NEGOTIATE.length()).trim(); + + if (tokenPart.startsWith("=")) { + tokenPart = tokenPart.substring(1).trim(); + } + + if (StringUtils.isNotBlank(tokenPart)) { + return Base64.getDecoder().decode(tokenPart); + } + } + } + + return null; + } + + private static void logSuccess(String service, AuthzAuditEvent auditEvent) { + if (LOG.isDebugEnabled()) { + LOG.debug("Audit event posted to ingestor for service={} user={}", service, auditEvent.getUser()); + } + } + + private static GSSContext createSpnegoContext(String host) throws Exception { + GSSManager manager = GSSManager.getInstance(); + GSSName server = manager.createName("HTTP@" + host, GSSName.NT_HOSTBASED_SERVICE); + Oid mech = resolveSpnegoMechanism(manager); + + GSSContext context = manager.createContext(server, mech, null, GSSContext.DEFAULT_LIFETIME); + + context.requestMutualAuth(true); + context.requestCredDeleg(false); + + return context; + } + + private static Oid resolveSpnegoMechanism(GSSManager manager) throws Exception { + Oid spnego = new Oid("1.3.6.1.5.5.14"); + + for (Oid mech : manager.getMechs()) { + if (spnego.equals(mech)) { + return spnego; + } + } + + return new Oid("1.2.840.113554.1.2.2"); + } + + private static final class HttpPostResult { + private final int status; + private final String wwwAuthenticate; + + private HttpPostResult(int status, String wwwAuthenticate) { + this.status = status; + this.wwwAuthenticate = wwwAuthenticate; + } + } +} diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java index 7784a5f9242..c9663ed261b 100644 --- a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java +++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java @@ -3,8 +3,8 @@ * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance + * to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -18,29 +18,43 @@ */ package org.apache.ranger.authorization.elasticsearch.authorizer; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.ranger.audit.model.AuthzAuditEvent; -import org.apache.ranger.plugin.audit.RangerMultiResourceAuditHandler; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; -public class RangerElasticsearchAuditHandler extends RangerMultiResourceAuditHandler { +public class RangerElasticsearchAuditHandler extends RangerDefaultAuditHandler { + private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuditHandler.class); + private static final String PROP_ES_PLUGIN_AUDIT_EXCLUDED_USERS = "ranger.elasticsearch.plugin.audit.excluded.users"; private static final String PROP_ES_PLUGIN_AUDIT_INDEX = "xasecure.audit.destination.elasticsearch.index"; + private static final String PROP_AUDITSERVER_URL = "xasecure.audit.destination.auditserver.url"; private final String indexName; private final List excludeUsers; + private final String auditServerUrl; public RangerElasticsearchAuditHandler(Configuration config) { + super(config); + String esUser = "elasticsearch"; String excludeUserList = config.get(PROP_ES_PLUGIN_AUDIT_EXCLUDED_USERS, esUser); - excludeUsers = Arrays.asList(excludeUserList.split(",")); - indexName = config.get(PROP_ES_PLUGIN_AUDIT_INDEX, "ranger_audits"); + excludeUsers = Arrays.asList(excludeUserList.split(",")); + indexName = config.get(PROP_ES_PLUGIN_AUDIT_INDEX, "ranger_audits"); + auditServerUrl = config.get(PROP_AUDITSERVER_URL); + + ElasticsearchAuditIngestorClient.init(config); } @Override @@ -53,15 +67,44 @@ public void processResult(RangerAccessResult result) { AuthzAuditEvent auditEvent = super.getAuthzEvents(result); + if (auditEvent == null) { + return; + } + + if (!logAuditEvent(auditEvent)) { + MiscUtil.logErrorMessageByInterval(LOG, "fail to log audit event " + auditEvent); + } + } + + private boolean logAuditEvent(AuthzAuditEvent auditEvent) { + if (StringUtils.isNotBlank(auditServerUrl) && ElasticsearchAuditIngestorClient.post(auditServerUrl, auditEvent)) { + return true; + } + + if (AuditProviderFactory.getInstance().logOnRequestThread(auditEvent)) { + return true; + } + super.logAuthzAudit(auditEvent); + + return true; } private boolean isAuditingNeeded(final RangerAccessResult result) { + if (result == null) { + return false; + } + + RangerAccessRequest request = result.getAccessRequest(); + + if (request == null) { + return false; + } + boolean ret = true; boolean isAllowed = result.getIsAllowed(); - RangerAccessRequest request = result.getAccessRequest(); RangerAccessResourceImpl resource = (RangerAccessResourceImpl) request.getResource(); - String resourceName = (String) resource.getValue("index"); + String resourceName = resource == null ? null : (String) resource.getValue("index"); String requestUser = request.getUser(); if (resourceName != null && resourceName.equals(indexName) && excludeUsers.contains(requestUser) && isAllowed) { diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java index f05ab9ff025..f0dd4e041be 100644 --- a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java +++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; @@ -30,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -39,9 +41,16 @@ public class RangerElasticsearchAuthorizer implements RangerElasticsearchAccessC private static volatile RangerElasticsearchInnerPlugin elasticsearchPlugin; + private final String configDir; + public RangerElasticsearchAuthorizer() { + this(null); + } + + public RangerElasticsearchAuthorizer(String configDir) { LOG.debug("==> RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()"); + this.configDir = configDir; this.init(); LOG.debug("<== RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()"); @@ -57,7 +66,7 @@ public void init() { plugin = elasticsearchPlugin; if (plugin == null) { - plugin = new RangerElasticsearchInnerPlugin(); + plugin = new RangerElasticsearchInnerPlugin(configDir); plugin.init(); @@ -95,8 +104,23 @@ public boolean checkPermission(String user, List groups, String index, S } static class RangerElasticsearchInnerPlugin extends RangerBasePlugin { - public RangerElasticsearchInnerPlugin() { - super("elasticsearch", "elasticsearch"); + public RangerElasticsearchInnerPlugin(String configDir) { + super(createPluginConfig(configDir)); + } + + private static RangerPluginConfig createPluginConfig(String configDir) { + List additionalConfigFiles = null; + + if (configDir != null) { + File dir = new File(configDir); + + additionalConfigFiles = new ArrayList<>(); + additionalConfigFiles.add(new File(dir, "ranger-elasticsearch-audit.xml")); + additionalConfigFiles.add(new File(dir, "ranger-elasticsearch-security.xml")); + additionalConfigFiles.add(new File(dir, "ranger-policymgr-ssl.xml")); + } + + return new RangerPluginConfig("elasticsearch", null, "elasticsearch", null, null, additionalConfigFiles, null); } @Override diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java index f3d07c0b9fc..88280597e9e 100644 --- a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java +++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java @@ -40,7 +40,7 @@ public class RangerServiceElasticsearch extends RangerBaseService { public static final String ACCESS_TYPE_READ = "read"; - private RangerServiceElasticsearch() { + public RangerServiceElasticsearch() { super(); } diff --git a/pom.xml b/pom.xml index 61140d18024..02198d72d28 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ https://repository.apache.org/service/local/staging/deploy/maven2 3.6.2 2.7.12 + 2.14.2 7.17.29 2.15.0 2.18.9 diff --git a/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties b/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties index 95f9604aa95..1af4c62f6f1 100644 --- a/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties +++ b/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties @@ -56,10 +56,7 @@ elasticsearch.version=${elasticsearch.version} ### optional elements for plugins: # # 'extended.plugins': other plugins this plugin extends through SPI -#extended.plugins=${extendedPlugins} +extended.plugins=x-pack-security # # 'has.native.controller': whether or not the plugin has a native controller has.native.controller=false -# -# 'requires.keystore': whether or not the plugin needs the elasticsearch keystore be created -requires.keystore=false diff --git a/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy b/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy index 4f96d87f5d7..b4de434f20e 100644 --- a/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy +++ b/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy @@ -9,31 +9,44 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +// Bare grant applies to all jars in this plugin directory (shim, impl, Hadoop, Ranger). +// ES 7.17 rejects AllPermission here; permissions must match PolicyUtil.ALLOWED_PLUGIN_PERMISSIONS. grant { - permission java.lang.RuntimePermission "createClassLoader"; + // Hadoop Configuration / UGI static init (same as repository-hdfs plugin) permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.RuntimePermission "setContextClassLoader"; - permission java.lang.RuntimePermission "shutdownHooks"; permission java.lang.RuntimePermission "accessDeclaredMembers"; - permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.util.PropertyPermission "*", "read,write"; - permission javax.security.auth.AuthPermission "getLoginConfiguration"; - permission javax.security.auth.AuthPermission "setLoginConfiguration"; + // JAAS / Kerberos (Hadoop auth stack) + permission javax.security.auth.AuthPermission "getSubject"; + permission javax.security.auth.AuthPermission "doAs"; + permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.security.krb5"; + permission javax.security.auth.AuthPermission "modifyPrivateCredentials"; + permission javax.security.auth.AuthPermission "modifyPrincipals"; + permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read"; + permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read"; + permission java.lang.RuntimePermission "loadLibrary.jaas"; + permission java.lang.RuntimePermission "loadLibrary.jaas_unix"; + permission java.lang.RuntimePermission "loadLibrary.jaas_nt"; + permission javax.security.auth.AuthPermission "modifyPublicCredentials"; + permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer"; + permission java.security.SecurityPermission "insertProvider"; + permission javax.security.auth.kerberos.ServicePermission "*", "initiate"; - permission java.net.NetPermission "getProxySelector"; - // adapt to connect different IP and Port + // Ranger plugin: policy download from Admin, audit ingestor HTTP POST + permission java.net.SocketPermission "ranger-audit-ingestor.rangernw:7081", "connect,resolve"; + permission java.net.SocketPermission "ranger.rangernw:6080", "connect,resolve"; permission java.net.SocketPermission "*", "connect,resolve"; - - permission java.util.PropertyPermission "*", "read,write"; - // adapt to different directories configured by user - permission java.io.FilePermission "<>", "read,write"; + permission java.net.SocketPermission "localhost:0", "listen,resolve"; + permission java.io.FilePermission "<>", "read"; }; diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java deleted file mode 100644 index c5290013c5d..00000000000 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.authorization.elasticsearch.authorizer; - -import org.apache.ranger.plugin.classloader.RangerPluginClassLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class RangerElasticsearchAuthorizer { - private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizer.class); - - private static final String RANGER_PLUGIN_TYPE = "elasticsearch"; - private static final String RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME = "org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer"; - - private RangerPluginClassLoader rangerPluginClassLoader; - private ClassLoader esClassLoader; - private RangerElasticsearchAccessControl rangerElasticsearchAccessControl; - - public RangerElasticsearchAuthorizer() { - LOG.debug("==> RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()"); - - this.init(); - - LOG.debug("<== RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()"); - } - - public void init() { - LOG.debug("==> RangerElasticsearchAuthorizer.init()"); - - try { - // In elasticsearch this.getClass().getClassLoader() is FactoryURLClassLoader, - // but Thread.currentThread().getContextClassLoader() is AppClassLoader. - esClassLoader = Thread.currentThread().getContextClassLoader(); - - Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader()); - - rangerPluginClassLoader = RangerPluginClassLoader.getInstance(RANGER_PLUGIN_TYPE, this.getClass()); - - Thread.currentThread().setContextClassLoader(esClassLoader); - - @SuppressWarnings("unchecked") - Class cls = (Class) Class.forName(RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME, true, rangerPluginClassLoader); - - activatePluginClassLoader(); - - rangerElasticsearchAccessControl = cls.newInstance(); - } catch (Exception e) { - LOG.error("Error Enabling RangerElasticsearchAuthorizer", e); - } finally { - deactivatePluginClassLoader(); - } - - LOG.debug("<== RangerElasticsearchAuthorizer.init()"); - } - - public boolean checkPermission(String user, List groups, String index, String action, String clientIPAddress) { - boolean ret; - - LOG.debug("==> RangerElasticsearchAuthorizer.checkPermission()"); - - try { - activatePluginClassLoader(); - - ret = rangerElasticsearchAccessControl.checkPermission(user, groups, index, action, clientIPAddress); - } finally { - deactivatePluginClassLoader(); - } - - LOG.debug("<== RangerElasticsearchAuthorizer.checkPermission()"); - - return ret; - } - - private void activatePluginClassLoader() { - if (rangerPluginClassLoader != null) { - Thread.currentThread().setContextClassLoader(rangerPluginClassLoader); - } - } - - private void deactivatePluginClassLoader() { - if (esClassLoader != null) { - Thread.currentThread().setContextClassLoader(esClassLoader); - } - } -} diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java new file mode 100644 index 00000000000..c7e21041b61 --- /dev/null +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.elasticsearch.authorizer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.PrivilegedExceptionAction; +import java.util.List; + +/** + * Shim-side delegate that loads the real {@link RangerElasticsearchAuthorizer} implementation + * from the Elasticsearch plugin classloader. ES 7.17+ does not grant plugins + * {@code createClassLoader}, so we avoid {@code RangerPluginClassLoader} here and rely on + * impl jars being on the plugin classpath. + */ +public class RangerElasticsearchAuthorizerDelegate { + private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizerDelegate.class); + + private static final String RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME = + "org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer"; + + private RangerElasticsearchAccessControl rangerElasticsearchAccessControl; + + private final String configDir; + + public RangerElasticsearchAuthorizerDelegate(String configDir) { + LOG.debug("==> RangerElasticsearchAuthorizerDelegate()"); + + this.configDir = configDir; + this.init(); + + LOG.debug("<== RangerElasticsearchAuthorizerDelegate()"); + } + + public void init() { + LOG.debug("==> RangerElasticsearchAuthorizerDelegate.init()"); + + try { + @SuppressWarnings("unchecked") + Class cls = (Class) Class.forName( + RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME); + + rangerElasticsearchAccessControl = java.security.AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> cls.getDeclaredConstructor(String.class) + .newInstance(configDir)); + } catch (Exception e) { + LOG.error("Error Enabling RangerElasticsearchAuthorizer", e); + } + + LOG.debug("<== RangerElasticsearchAuthorizerDelegate.init()"); + } + + public boolean checkPermission(String user, List groups, String index, String action, String clientIPAddress) { + LOG.debug("==> RangerElasticsearchAuthorizerDelegate.checkPermission()"); + + if (rangerElasticsearchAccessControl == null) { + LOG.warn("RangerElasticsearchAuthorizer is not initialized; denying access."); + + return false; + } + + boolean ret; + + try { + ret = java.security.AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> rangerElasticsearchAccessControl.checkPermission( + user, groups, index, action, clientIPAddress)); + } catch (Exception e) { + LOG.error("Error checking Ranger permission for user[{}] action[{}] index[{}]", user, action, index, e); + + ret = false; + } + + LOG.debug("<== RangerElasticsearchAuthorizerDelegate.checkPermission()"); + + return ret; + } +} diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java index e3b740f00b8..cbe1809a9cc 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java @@ -17,8 +17,8 @@ package org.apache.ranger.authorization.elasticsearch.plugin; +import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizerDelegate; import org.apache.ranger.authorization.elasticsearch.plugin.action.filter.RangerSecurityActionFilter; -import org.apache.ranger.authorization.elasticsearch.plugin.rest.filter.RangerSecurityRestFilter; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -31,7 +31,6 @@ import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; @@ -39,17 +38,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.function.Supplier; -import java.util.function.UnaryOperator; +/** + * Ranger authorization integrates with X-Pack Security: authentication and REST handling + * are delegated to x-pack-security; Ranger enforces policies via {@link RangerSecurityActionFilter} + * using the verified user from {@link org.elasticsearch.xpack.core.security.SecurityContext}. + */ public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin { private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchPlugin.class); @@ -70,57 +70,50 @@ public List getActionFilters() { return Collections.singletonList(rangerSecurityActionFilter); } - @Override - public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { - return handler -> new RangerSecurityRestFilter(settings, threadContext, handler); - } - @Override public Collection createComponents(final Client client, final ClusterService clusterService, final ThreadPool threadPool, final ResourceWatcherService resourceWatcherService, final ScriptService scriptService, final NamedXContentRegistry xContentRegistry, final Environment environment, final NodeEnvironment nodeEnvironment, final NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { - addPluginConfig2Classpath(environment); + Path configPath = registerPluginConfigDir(environment); - rangerSecurityActionFilter = new RangerSecurityActionFilter(settings, threadPool.getThreadContext()); + ThreadContext threadContext = threadPool.getThreadContext(); + + RangerElasticsearchAuthorizerDelegate authorizer = initAuthorizer(threadContext, configPath); + + rangerSecurityActionFilter = new RangerSecurityActionFilter(settings, threadContext, authorizer); return Collections.singletonList(rangerSecurityActionFilter); } /** - * Add ranger elasticsearch plugin config directory to classpath, - * then the plugin can load its configuration files from classpath. + * Initialize Ranger in Elasticsearch system context so Hadoop/Ranger plugin setup is not + * blocked by the plugin security manager during node startup. */ - private void addPluginConfig2Classpath(Environment environment) { - Path configPath = environment.configFile().resolve(RANGER_ELASTICSEARCH_PLUGIN_CONF_NAME); + private RangerElasticsearchAuthorizerDelegate initAuthorizer(ThreadContext threadContext, Path configPath) { + String configDir = configPath != null ? configPath.toAbsolutePath().toString() : null; - if (configPath == null) { - LOG.error("Failed to add ranger elasticsearch plugin config directory [ranger-elasticsearch-plugin] to classpath."); + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { + threadContext.markAsSystemContext(); - return; + return new RangerElasticsearchAuthorizerDelegate(configDir); } + } - File configFile = configPath.toFile(); - - try { - if (configFile.exists()) { - ClassLoader classLoader = this.getClass().getClassLoader(); + /** + * Resolve the on-disk Ranger config directory for the authorizer implementation. + * ES 7.17 on Java 17 cannot extend the plugin classloader via reflection. + */ + private Path registerPluginConfigDir(Environment environment) { + Path configPath = environment.configFile().resolve(RANGER_ELASTICSEARCH_PLUGIN_CONF_NAME); - // This classLoader is FactoryURLClassLoader in elasticsearch - if (classLoader instanceof URLClassLoader) { - URLClassLoader urlClassLoader = (URLClassLoader) classLoader; - Class urlClass = urlClassLoader.getClass(); - Method method = urlClass.getSuperclass().getDeclaredMethod("addURL", URL.class); + if (!Files.isDirectory(configPath)) { + LOG.error("Ranger elasticsearch plugin config directory [{}] does not exist.", configPath); - method.setAccessible(true); - method.invoke(urlClassLoader, configFile.toURI().toURL()); + return null; + } - LOG.info("Success to add ranger elasticsearch plugin config directory [{}] to classpath.", configFile.getCanonicalPath()); - } - } - } catch (Exception e) { - LOG.error("Failed to add ranger elasticsearch plugin config directory [ranger-elasticsearch-plugin] to classpath.", e); + LOG.info("Using Ranger elasticsearch plugin config directory [{}].", configPath); - throw new RuntimeException(e); - } + return configPath; } } diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java index 1d70e3b0a1c..8c63a1f5dd0 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java @@ -18,7 +18,7 @@ package org.apache.ranger.authorization.elasticsearch.plugin.action.filter; import org.apache.commons.lang3.StringUtils; -import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer; +import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizerDelegate; import org.apache.ranger.authorization.elasticsearch.plugin.authc.ElasticsearchAuthenticatedUserResolver; import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken; import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils; @@ -33,37 +33,39 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; public class RangerSecurityActionFilter extends AbstractLifecycleComponent implements ActionFilter { - private static final Logger LOG = LoggerFactory.getLogger(RangerSecurityActionFilter.class); - private final Settings settings; private final ThreadContext threadContext; - private final RangerElasticsearchAuthorizer rangerElasticsearchAuthorizer = new RangerElasticsearchAuthorizer(); + private final RangerElasticsearchAuthorizerDelegate rangerElasticsearchAuthorizer; - public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext) { + public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext, RangerElasticsearchAuthorizerDelegate rangerElasticsearchAuthorizer) { super(); - this.settings = settings; - this.threadContext = threadContext; + this.settings = settings; + this.threadContext = threadContext; + this.rangerElasticsearchAuthorizer = rangerElasticsearchAuthorizer; } + /** + * Run after x-pack {@code SecurityActionFilter} (order {@code Integer.MIN_VALUE}) so + * {@link org.elasticsearch.xpack.core.security.SecurityContext} holds the verified user. + */ @Override public int order() { - return 0; + return Integer.MAX_VALUE; } @Override public void apply(Task task, String action, Request request, ActionListener listener, ActionFilterChain chain) { String user = threadContext.getTransient(UsernamePasswordToken.USERNAME); - if (StringUtils.isEmpty(user)) { - ElasticsearchAuthenticatedUserResolver authResolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + ElasticsearchAuthenticatedUserResolver authResolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + List groups = null; + if (StringUtils.isEmpty(user)) { if (authResolver.requiresAuthenticatedUser()) { user = authResolver.resolveUsername(); @@ -73,12 +75,25 @@ public void app } } + if (StringUtils.isNotEmpty(user)) { + List roles = authResolver.resolveRoles(); + + if (!roles.isEmpty()) { + groups = roles; + } + } + + if (shouldBypassRangerCheck(user, action)) { + chain.proceed(task, action, request, listener); + return; + } + if (StringUtils.isNotEmpty(user)) { List indexs = RequestUtils.getIndexFromRequest(request); String clientIPAddress = threadContext.getTransient(RequestUtils.CLIENT_IP_ADDRESS); for (String index : indexs) { - boolean result = rangerElasticsearchAuthorizer.checkPermission(user, null, index, action, clientIPAddress); + boolean result = rangerElasticsearchAuthorizer.checkPermission(user, groups, index, action, clientIPAddress); if (!result) { String errorMsg = "Error: User[{}] could not do action[{}] on index[{}]"; @@ -86,8 +101,6 @@ public void app throw new ElasticsearchStatusException(errorMsg, RestStatus.FORBIDDEN, user, action, index); } } - } else if (threadContext.isSystemContext()) { - LOG.debug("System context request, skipping Ranger permission check for action[{}].", action); } else { throw new ElasticsearchStatusException("Error: Request requires authenticated user.", RestStatus.UNAUTHORIZED); } @@ -95,6 +108,18 @@ public void app chain.proceed(task, action, request, listener); } + private boolean shouldBypassRangerCheck(String user, String action) { + if (threadContext.isSystemContext()) { + return true; + } + + if (StringUtils.isNotEmpty(user) && user.charAt(0) == '_') { + return true; + } + + return StringUtils.isNotEmpty(action) && action.startsWith("internal:"); + } + @Override protected void doStart() { } diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java index 823d9502ef8..a84834221ed 100644 --- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java +++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/ElasticsearchAuthenticatedUserResolver.java @@ -17,12 +17,17 @@ package org.apache.ranger.authorization.elasticsearch.plugin.authc; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.user.User; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * Resolves the Elasticsearch-verified user for the current request. * Caller identity must come from X-Pack Security, not from client-supplied headers. @@ -53,4 +58,28 @@ public String resolveUsername() { return StringUtils.isEmpty(username) ? null : username; } + + /** + * Returns X-Pack role names for the verified user. These are passed to Ranger as group + * hints so role/group policies can bind when Hadoop UGI has no group membership. + */ + public List resolveRoles() { + if (!requiresAuthenticatedUser()) { + return Collections.emptyList(); + } + + User user = securityContext.getUser(); + + if (user == null) { + return Collections.emptyList(); + } + + String[] roles = user.roles(); + + if (ArrayUtils.isEmpty(roles)) { + return Collections.emptyList(); + } + + return Arrays.asList(roles); + } } diff --git a/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java index c3d0f9a4240..7e51b9e703e 100644 --- a/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java +++ b/ranger-elasticsearch-plugin-shim/src/test/java/org/apache/ranger/authorization/elasticsearch/plugin/authc/TestElasticsearchAuthenticatedUserResolver.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Arrays; + class TestElasticsearchAuthenticatedUserResolver { @Test void resolveUsernameReturnsVerifiedUser() { @@ -50,6 +52,28 @@ void resolveUsernameReturnsNullWithoutVerifiedUser() { Assertions.assertNull(resolver.resolveUsername()); } + @Test + void resolveRolesReturnsVerifiedUserRoles() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + SecurityContext securityContext = new SecurityContext(settings, threadContext); + + securityContext.setUser(new User("admin", "superuser", "kibana_user"), Version.CURRENT); + + ElasticsearchAuthenticatedUserResolver resolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + Assertions.assertEquals(Arrays.asList("superuser", "kibana_user"), resolver.resolveRoles()); + } + + @Test + void resolveRolesReturnsEmptyWithoutVerifiedUser() { + Settings settings = Settings.builder().build(); + ThreadContext threadContext = new ThreadContext(settings); + ElasticsearchAuthenticatedUserResolver resolver = new ElasticsearchAuthenticatedUserResolver(settings, threadContext); + + Assertions.assertTrue(resolver.resolveRoles().isEmpty()); + } + @Test void systemContextDoesNotRequireAuthenticatedUser() { Settings settings = Settings.builder().build(); diff --git a/ranger_in_docker b/ranger_in_docker index 0db00f9aa0b..ecbdb89ab11 100755 --- a/ranger_in_docker +++ b/ranger_in_docker @@ -42,7 +42,7 @@ # by defining comma separated optional services as following: # ENABLED_RANGER_SERVICES="hadoop,hive,hbase,knox,kms" # List of optional services: -# tagsync,hadoop,hbase,kafka,hive,knox,kms +# tagsync,hadoop,hbase,kafka,hive,knox,kms,elasticsearch # if [ -z "${RANGER_HOME}" ] then