diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index 58b136b851..cbbdf591ba 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -62,6 +62,18 @@ private AuditServerConstants() {} public static final String PROP_BUFFER_PARTITIONS = "kafka.topic.partitions.buffer"; public static final String PROP_REPLICATION_FACTOR = "kafka.replication.factor"; + // Dynamic partition plan (Kafka compacted registry topic) + public static final String PROP_PARTITION_PLAN_TOPIC = "kafka.partition.plan.topic"; + public static final String PROP_PARTITION_PLAN_REFRESH_INTERVAL_MS = "kafka.partition.plan.refresh.interval.ms"; + public static final String PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS = "kafka.partition.plan.consumer.poll.timeout.ms"; + public static final String PROP_PARTITION_PLAN_DYNAMIC_ENABLED = "kafka.partition.plan.dynamic.enabled"; + public static final String PROP_PARTITION_PLAN_ALLOWED_USERS = "kafka.partition.plan.allowed.users"; + public static final String DEFAULT_PARTITION_PLAN_TOPIC = "ranger_audit_partition_plan"; + public static final int DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS = 30000; + public static final int DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS = 500; + public static final int PARTITION_PLAN_TOPIC_PARTITION_COUNT = 1; + public static final String KAFKA_TOPIC_CLEANUP_POLICY_COMPACT = "compact"; + // Kafka producer tuning (ranger.audit.ingestor.kafka.producer.*) public static final String PROP_KAFKA_PRODUCER_PREFIX = "kafka.producer."; public static final String PROP_PRODUCER_BATCH_SIZE = "batch.size"; @@ -108,9 +120,9 @@ private AuditServerConstants() {} public static final long DEFAULT_OFFSET_COMMIT_INTERVAL_MS = 30000; // 30 seconds public static final int DEFAULT_MAX_POLL_RECORDS = 500; // Kafka default batch size - // Default configured plugins: each gets allocated partitions from the topic + // Empty by default: operators opt in via XML (static) or REST (dynamic). See ranger-audit-ingestor-site.xml. public static final String DEFAULT_PARTITIONER_CLASS = "org.apache.ranger.audit.producer.kafka.AuditPartitioner"; - public static final String DEFAULT_CONFIGURED_PLUGINS = "hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi"; + public static final String DEFAULT_CONFIGURED_PLUGINS = ""; public static final short DEFAULT_REPLICATION_FACTOR = 3; public static final int DEFAULT_TOPIC_PARTITIONS = 10; public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3; diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java index bfba27260a..f1e8312576 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java @@ -27,6 +27,7 @@ import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.errors.TopicExistsException; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConstants; import org.slf4j.Logger; @@ -39,6 +40,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ExecutionException; public class AuditMessageQueueUtils { private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueueUtils.class); @@ -50,30 +52,13 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP LOG.info("==> AuditMessageQueueUtils:createAuditsTopicIfNotExists(propPrefix={})", propPrefix); String ret = null; - String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); - String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); - String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); - String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM); - int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000); - int partitions = getPartitions(props, propPrefix); - short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); - int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000); - int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; - int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); + String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + int partitions = getPartitions(props, propPrefix); + short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); + int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; + int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); - Map kafkaProp = new HashMap<>(); - - kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - kafkaProp.put("sasl.mechanism", saslMechanism); - kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); - - if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) { - kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix)); - kafkaProp.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); - } - - kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS); - kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS); + Map kafkaProp = buildAdminClientConfig(props, propPrefix); for (int currentAttempt = 1; currentAttempt <= maxAttempts && ret == null; currentAttempt++) { try (AdminClient admin = AdminClient.create(kafkaProp)) { @@ -92,7 +77,7 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP LOG.info("Topic '{}' configs: {}", topicName, topicConfigs); } - admin.createTopics(Collections.singletonList(topic)).all().get(); + createTopicIgnoringAlreadyExists(admin, topic); ret = topic.name(); @@ -143,6 +128,116 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP return ret; } + /** + * Create the compacted partition-plan registry topic if missing. + * Always exactly one partition ({@link AuditServerConstants#PARTITION_PLAN_TOPIC_PARTITION_COUNT}); + * partition count is not configurable. A single partition is required so every plan version compacts + * under one key on one log end. Safe when multiple ingestor pods start together (concurrent topic creation). + */ + public static String createPartitionPlanTopicIfNotExists(Properties props, String propPrefix) { + String planTopic = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); + int planTopicPartitions = AuditServerConstants.PARTITION_PLAN_TOPIC_PARTITION_COUNT; + int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; + int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try (AdminClient admin = AdminClient.create(adminConfig)) { + Set topicNames = admin.listTopics().names().get(); + if (!topicNames.contains(planTopic)) { + LOG.info("Creating partition plan topic '{}' with {} partition and replication factor {}", planTopic, planTopicPartitions, replicationFactor); + NewTopic topic = new NewTopic(planTopic, planTopicPartitions, replicationFactor); + topic.configs(Collections.singletonMap("cleanup.policy", AuditServerConstants.KAFKA_TOPIC_CLEANUP_POLICY_COMPACT)); + createTopicIgnoringAlreadyExists(admin, topic); + AuditServerUtils.waitUntilTopicReady(admin, planTopic, Duration.ofSeconds(60)); + int partitionCount = describeTopicPartitionCount(admin, planTopic); + LOG.info("Partition plan topic '{}' is ready with {} partition(s)", planTopic, partitionCount); + } else { + int partitionCount = describeTopicPartitionCount(admin, planTopic); + requirePlanTopicPartitionCount(planTopic, partitionCount, planTopicPartitions); + LOG.info("Partition plan topic '{}' already exists with {} partition(s)", planTopic, partitionCount); + } + return planTopic; + } catch (Exception ex) { + if (attempt < maxAttempts) { + LOG.warn("Failed to ensure partition plan topic on attempt {}/{}. Retrying in {} ms. Error: {}", attempt, maxAttempts, retryDelayMs, ex.getMessage()); + try { + Thread.sleep(retryDelayMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } else { + LOG.error("Failed to create partition plan topic '{}' after {} attempts", planTopic, attempt, ex); + throw new RuntimeException("Failed to create partition plan topic '" + planTopic + "' after " + attempt + " attempts", ex); + } + } + } + throw new RuntimeException("Failed to create partition plan topic '" + planTopic + "'"); + } + + /** + * Returns whether the compacted partition-plan registry topic already exists on the audit Kafka cluster. + * When the check fails (broker unreachable, ACL denied), returns {@code false} so callers can fall back + * to static XML {@code auth_to_local} rules until the plan topic is available. + */ + public static boolean partitionPlanTopicExists(Properties props, String propPrefix) { + String planTopic = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + Set topicNames = admin.listTopics().names().get(); + boolean exists = topicNames.contains(planTopic); + LOG.debug("Partition plan topic '{}' exists: {}", planTopic, exists); + return exists; + } catch (Exception ex) { + LOG.warn("Could not determine whether partition plan topic '{}' exists: {}. Assuming it does not.", planTopic, ex.getMessage()); + return false; + } + } + + public static Map buildAdminClientConfig(Properties props, String propPrefix) { + String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM); + int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000); + int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000); + + Map kafkaProp = new HashMap<>(); + kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + kafkaProp.put("sasl.mechanism", saslMechanism); + kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) { + kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix)); + kafkaProp.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + } + kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS); + kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS); + return kafkaProp; + } + + private static int describeTopicPartitionCount(AdminClient admin, String topicName) throws Exception { + TopicDescription topicDescription = admin.describeTopics(Collections.singletonList(topicName)).values().get(topicName).get(); + return topicDescription.partitions().size(); + } + + private static void requirePlanTopicPartitionCount(String planTopic, int actualPartitions, int expectedPartitions) { + if (actualPartitions != expectedPartitions) { + throw new RuntimeException("Partition plan topic '" + planTopic + "' must have exactly " + expectedPartitions + " partition(s) for compacted registry semantics, but has " + actualPartitions); + } + } + + private static void createTopicIgnoringAlreadyExists(AdminClient admin, NewTopic topic) throws Exception { + try { + admin.createTopics(Collections.singletonList(topic)).all().get(); + } catch (ExecutionException ex) { + if (!(ex.getCause() instanceof TopicExistsException)) { + throw ex; + } + LOG.info("Topic '{}' already exists", topic.name()); + } + } + public static String getJAASConfig(Properties props, String propPrefix) { // Use ranger service principal and keytab for Kafka authentication // This ensures consistent identity across all Ranger services and destination writes @@ -215,6 +310,22 @@ public static String getJAASConfig(Properties props, String propPrefix) { return jaasConfig; } + /** Grows the audit topic partition count when the plan requires more partitions than exist today. */ + public static void ensureTopicPartitionCount(Properties props, String propPrefix, String topicName, int requiredPartitions) { + LOG.info("Ensuring topic '{}' has at least {} partitions", topicName, requiredPartitions); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + updateExistingTopicPartitions(admin, topicName, requiredPartitions); + LOG.info("Topic '{}' partition count satisfied (required >= {})", topicName, requiredPartitions); + } catch (Exception e) { + LOG.error("Failed to ensure partition count for topic '{}' (required >= {})", topicName, requiredPartitions, e); + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException("Failed to ensure partition count for topic '" + topicName + "'", e); + } + } + private static String updateExistingTopicPartitions(AdminClient admin, String topicName, int partitions) { LOG.info("==> AuditMessageQueueUtils:updateExistingTopicPartitions() topic: {}, desired partitions: {}", topicName, partitions); diff --git a/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java index e20a696474..950a7d477f 100644 --- a/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java +++ b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.ranger.audit.utils; +import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.Test; @@ -65,4 +66,16 @@ public void testBuildTopicConfigsMapsAllSetProperties() { assertEquals("lz4", configs.get("compression.type")); assertEquals("2", configs.get("min.insync.replicas")); } + + @Test + public void testBuildAdminClientConfigUsesBootstrapServers() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS, "kafka:9092"); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, "PLAINTEXT"); + + Map adminConfig = AuditMessageQueueUtils.buildAdminClientConfig(props, PROP_PREFIX); + + assertEquals("kafka:9092", adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG)); + assertEquals("PLAINTEXT", adminConfig.get(AdminClientConfig.SECURITY_PROTOCOL_CONFIG)); + } } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java index 2168409896..800ebda4ea 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java @@ -23,6 +23,8 @@ import org.apache.ranger.audit.destination.AuditDestination; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanWatcher; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.utils.AuditMessageQueueUtils; import org.slf4j.Logger; @@ -48,6 +50,7 @@ public class AuditMessageQueue extends AuditDestination { private String topicName; private Thread producerThread; private AuditRecoveryManager recoveryManager; + private PartitionPlanWatcher partitionPlanWatcher; @Override public void init(Properties props, String propPrefix) { @@ -56,6 +59,7 @@ public void init(Properties props, String propPrefix) { super.init(props, propPrefix); createAuditsTopic(props, PROP_INGESTOR_PREFIX); + startPartitionPlanWatcherIfEnabled(props); createKafkaProducer(props, PROP_INGESTOR_PREFIX); createRecoveryManager(props, PROP_INGESTOR_PREFIX); @@ -75,6 +79,16 @@ public void start() { public void stop() { LOG.info("==> AuditMessageQueue.stop() [CORE AUDIT SERVER]"); + if (partitionPlanWatcher != null) { + try { + partitionPlanWatcher.stop(); + } catch (Exception e) { + LOG.error("Error stopping partition plan watcher", e); + } finally { + partitionPlanWatcher = null; + } + } + // Shutdown recovery manager first to process any remaining messages if (recoveryManager != null) { try { @@ -340,6 +354,22 @@ private void createAuditsTopic(final Properties props, final String propPrefix) } } + private void startPartitionPlanWatcherIfEnabled(final Properties props) { + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(props, PROP_INGESTOR_PREFIX)) { + return; + } + if (topicName == null) { + throw new IllegalStateException("Audit topic must be created before starting partition plan watcher"); + } + try { + partitionPlanWatcher = new PartitionPlanWatcher(props, PROP_INGESTOR_PREFIX, topicName, null); + partitionPlanWatcher.startBlocking(); + } catch (Exception e) { + LOG.error("Failed to start partition plan watcher for audit topic '{}'", topicName, e); + throw new RuntimeException("Failed to start partition plan watcher for dynamic partitioning", e); + } + } + private void createRecoveryManager(final Properties props, final String propPrefix) { // Create recovery manager even if kafkaProducer is null - it will handle null producer gracefully // This ensures audits are spooled when Kafka is unavailable during startup diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java index d0832bdaa6..081e5a544e 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java @@ -22,6 +22,10 @@ import org.apache.kafka.clients.producer.Partitioner; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.PartitionInfo; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; import org.apache.ranger.audit.server.AuditServerConstants; import org.apache.ranger.audit.utils.AuditServerLogFormatter; import org.slf4j.Logger; @@ -50,11 +54,19 @@ public class AuditPartitioner implements Partitioner { private int[] configuredPluginPartitionEnd; private int bufferPartitionStart; private int bufferPartitionCount; + private boolean dynamicPlanEnabled; private final ConcurrentHashMap appIdCounters = new ConcurrentHashMap<>(); @Override public void configure(Map configs) { String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + String ingestorPropPrefix = propPrefix.substring(0, propPrefix.length() - 1); + dynamicPlanEnabled = PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configs, ingestorPropPrefix); + if (dynamicPlanEnabled) { + LOG.info("Dynamic partition plan enabled — routing from in-memory plan (PartitionPlanHolder)"); + logDynamicPlanConfiguration(configs, propPrefix); + return; + } String pluginsStr = getConfig(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); configuredPlugins = pluginsStr.split(","); @@ -116,13 +128,21 @@ public void configure(Map configs) { @Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { + String appId = key != null ? key.toString() : null; + if (dynamicPlanEnabled) { + int numPartitions = resolveTopicPartitionCount(cluster, topic); + if (appId == null || appId.isEmpty()) { + return Math.abs(System.identityHashCode(key) % numPartitions); + } + return partitionFromPlan(appId, numPartitions); + } + int numPartitions = totalPartitions; List partitions = cluster.partitionsForTopic(topic); if (partitions != null && !partitions.isEmpty()) { numPartitions = partitions.size(); } - String appId = key != null ? key.toString() : null; if (appId == null || appId.isEmpty()) { return Math.abs(System.identityHashCode(key) % numPartitions); } @@ -135,10 +155,8 @@ public int partition(String topic, Object key, byte[] keyBytes, Object value, by end = start; } int rangeSize = end - start + 1; - int subPartition = appIdCounters - .computeIfAbsent(appId, k -> new AtomicInteger(0)) - .getAndIncrement() % rangeSize; - return start + subPartition; + int roundRobinIndex = nextRoundRobinIndex(appId, rangeSize); + return start + roundRobinIndex; } else { // Unconfigured plugin - use buffer partitions int start = Math.min(bufferPartitionStart, numPartitions - 1); @@ -156,6 +174,164 @@ public void close() { appIdCounters.clear(); } + /** + * Routes one audit event to a Kafka partition using the in-memory dynamic partition plan. + * + *

The plan splits the audit topic into dedicated plugin lanes (configured plugins + * such as {@code hdfs}, {@code hiveServer}) and a shared buffer pool (everything else). + * Routing for a given {@code appId} follows this order: + *

    + *
  1. Known plugin with dedicated partitions — round-robin across that plugin's + * assignment list so load is spread evenly while preserving per-plugin ordering lanes. + * Example: {@code hdfs} → [0, 1, 2] sends three successive events to 0, then 1, + * then 2, then wraps.
    2. + *
  2. Unknown or unconfigured plugin — sticky hash into the shared buffer pool so + * the same {@code appId} always lands on the same buffer partition. + * Example: buffer → [10, 11]; {@code myCustomApp} consistently maps to 10 or 11.
    3. + *
  3. No buffer partitions in the plan — sticky hash across the full topic when every + * partition is assigned to configured plugins.
    4. + *
+ * + *

After topic scale-up, the Kafka producer's cluster metadata can lag behind + * {@link PartitionPlan#getTopicPartitionCount()}. We use the larger of the two counts as + * {@code effectiveTopicPartitionCount} so a newly planned tail id (e.g. 12) is not folded + * into the stale metadata ceiling (e.g. 11). + * + * @param appId plugin key from the audit event (Kafka record key) + * @param kafkaClusterPartitionCount partition count reported by live Kafka cluster metadata + * @return Kafka partition id to produce to + */ + private int partitionFromPlan(String appId, int kafkaClusterPartitionCount) { + PartitionPlan plan = PartitionPlanHolder.getInstance().getPlan(); + if (plan == null) { + LOG.error("Dynamic partition plan is not loaded; falling back to hash routing for appId '{}'", appId); + return hashAppIdToPartitionIndex(appId, kafkaClusterPartitionCount); + } + + int effectiveTopicPartitionCount = Math.max(kafkaClusterPartitionCount, plan.getTopicPartitionCount()); + + PluginPartitionAssignment pluginAssignment = findPluginAssignment(plan, appId); + if (pluginAssignment != null && !pluginAssignment.getPartitions().isEmpty()) { + List dedicatedPluginPartitions = pluginAssignment.getPartitions(); + int dedicatedLaneIndex = nextRoundRobinIndex(appId, dedicatedPluginPartitions.size()); + int plannedPartitionId = dedicatedPluginPartitions.get(dedicatedLaneIndex); + return resolvePlannedPartitionId(plannedPartitionId, effectiveTopicPartitionCount); + } + + List sharedBufferPartitions = plan.getBuffer().getPartitions(); + if (sharedBufferPartitions.isEmpty()) { + return hashAppIdToPartitionIndex(appId, effectiveTopicPartitionCount); + } + int bufferPoolIndex = hashAppIdToPartitionIndex(appId, sharedBufferPartitions.size()); + int plannedPartitionId = sharedBufferPartitions.get(bufferPoolIndex); + return resolvePlannedPartitionId(plannedPartitionId, effectiveTopicPartitionCount); + } + + /** + * Sticky hash: same {@code appId} always picks the same slot in {@code [0, slotCount)}. + * Used for buffer-pool routing and for plan-not-loaded fallback. + */ + private static int hashAppIdToPartitionIndex(String appId, int slotCount) { + return Math.floorMod(appId.hashCode(), slotCount); + } + + /** + * Returns the next dedicated-lane index for round-robin routing within one plugin's partition set. + * Each {@code appId} keeps its own counter (0, 1, 2, …); {@code % dedicatedLaneCount} cycles + * through that plugin's lanes only. + */ + private int nextRoundRobinIndex(String appId, int dedicatedLaneCount) { + AtomicInteger messageCounter = appIdCounters.computeIfAbsent(appId, k -> new AtomicInteger(0)); + return messageCounter.getAndIncrement() % dedicatedLaneCount; + } + + /** Looks up a plugin assignment using case-insensitive plugin id matching. */ + private static PluginPartitionAssignment findPluginAssignment(PartitionPlan plan, String appId) { + for (Map.Entry entry : plan.getPlugins().entrySet()) { + if (entry.getKey().equalsIgnoreCase(appId)) { + return entry.getValue(); + } + } + return null; + } + + /** Returns the live partition count for the audit topic from the Kafka cluster metadata. */ + private static int resolveTopicPartitionCount(Cluster cluster, String topic) { + List partitions = cluster.partitionsForTopic(topic); + if (partitions != null && !partitions.isEmpty()) { + return partitions.size(); + } + return 1; + } + + /** + * Converts a partition id from the plan into the id passed to the Kafka producer. + * + *

When cluster metadata is stale after scale-up, {@code effectiveTopicPartitionCount} may + * exceed what the broker metadata reports. Planned tail ids must still be returned as-is — + * never clamp partition 12 down to 11 just because metadata has not caught up yet. + * + * @param plannedPartitionId partition id from the dynamic plan assignment or buffer pool + * @param effectiveTopicPartitionCount {@code max(kafkaClusterPartitionCount, plan.topicPartitionCount)} + */ + private static int resolvePlannedPartitionId(int plannedPartitionId, int effectiveTopicPartitionCount) { + if (effectiveTopicPartitionCount <= 0) { + return 0; + } + if (plannedPartitionId < 0) { + return 0; + } + if (plannedPartitionId < effectiveTopicPartitionCount) { + return plannedPartitionId; + } + return plannedPartitionId; + } + + /** Logs the dynamic plan snapshot installed in {@link PartitionPlanHolder} at startup. */ + private void logDynamicPlanConfiguration(Map configs, String propPrefix) { + int defaultPerPlugin = getIntConfig(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + if (defaultPerPlugin < 1) { + defaultPerPlugin = 1; + } + + PartitionPlan plan = PartitionPlanHolder.getInstance().getPlan(); + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("****** AuditPartitioner Configuration ******"); + if (plan == null) { + logBuilder.add("Mode: ", "dynamic (plan not loaded yet)"); + logBuilder.logInfo(LOG); + return; + } + + logBuilder.add("Mode: ", "dynamic (PartitionPlanHolder)"); + logBuilder.add("Plan version: ", plan.getVersion()); + logBuilder.add("Total partitions: ", plan.getTopicPartitionCount()); + logBuilder.add("Configured plugins: ", plan.getPlugins().size()); + logBuilder.add("Default partitions per plugin: ", defaultPerPlugin); + for (Map.Entry entry : plan.getPlugins().entrySet()) { + List partitionIds = entry.getValue().getPartitions(); + String rangeInfo = formatPartitionRangeInfo(partitionIds); + logBuilder.add("Plugin '" + entry.getKey() + "'", rangeInfo); + } + logBuilder.add("Buffer partitions (unconfigured): ", formatBufferPartitionInfo(plan.getBuffer().getPartitions())); + logBuilder.logInfo(LOG); + } + + private static String formatPartitionRangeInfo(List partitionIds) { + if (partitionIds.isEmpty()) { + return "no partitions assigned in plan"; + } + return String.format("%d partitions (range: %d-%d)", + partitionIds.size(), partitionIds.get(0), partitionIds.get(partitionIds.size() - 1)); + } + + private static String formatBufferPartitionInfo(List bufferPartitionIds) { + if (bufferPartitionIds.isEmpty()) { + return "none (all topic partitions assigned to configured plugins)"; + } + return String.format("%d partitions (range: %d-%d)", + bufferPartitionIds.size(), bufferPartitionIds.get(0), bufferPartitionIds.get(bufferPartitionIds.size() - 1)); + } + private int indexOfConfiguredPlugin(String appId) { for (int i = 0; i < configuredPlugins.length; i++) { if (configuredPlugins[i].equalsIgnoreCase(appId)) { diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java new file mode 100644 index 0000000000..a4314b7e15 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** Parses and composes Kerberos {@code auth_to_local} rules from {@code ranger.audit.ingestor.auth.to.local}. */ +public class AuthToLocalRuleCatalog { + private static final Pattern RULE_SUBSTITUTION_SHORT_NAME_PATTERN = + Pattern.compile("s/\\.\\*/([^/]+)/\\s*$"); + private static final String GENERATED_SHORT_NAME_RULE_TEMPLATE = + "RULE:[2:$1/$2@$0](%s/.*@.*)s/.*/%s/"; + + private final List primaryCatalogRulesInOrder; + private final List fallbackRuleLines; + + AuthToLocalRuleCatalog(List primaryCatalogRulesInOrder, List fallbackRuleLines) { + this.primaryCatalogRulesInOrder = List.copyOf(primaryCatalogRulesInOrder); + this.fallbackRuleLines = List.copyOf(fallbackRuleLines); + } + + public static AuthToLocalRuleCatalog parse(String rawAuthToLocalRules) { + List primaryCatalogRulesInOrder = new ArrayList<>(); + List fallbackRuleLines = new ArrayList<>(); + for (String ruleLine : tokenizeRuleLines(rawAuthToLocalRules)) { + if (isFallbackKerberosRule(ruleLine)) { + fallbackRuleLines.add(ruleLine); + } else { + primaryCatalogRulesInOrder.add(new PrimaryCatalogRule(ruleLine, extractMappedShortName(ruleLine))); + } + } + return new AuthToLocalRuleCatalog(primaryCatalogRulesInOrder, fallbackRuleLines); + } + + /** Full static rule set (all primary catalog rules in order + fallback tail). */ + public String composeFullCatalogRules() { + List composedRuleLines = new ArrayList<>(primaryCatalogRulesInOrder.size() + fallbackRuleLines.size()); + for (PrimaryCatalogRule catalogRule : primaryCatalogRulesInOrder) { + composedRuleLines.add(catalogRule.ruleLine); + } + composedRuleLines.addAll(fallbackRuleLines); + return joinRuleLines(composedRuleLines); + } + + /** + * Builds the active rule set for dynamic mode: catalog rules whose mapped short name is in the union, + * plus generated rules for union members not covered by the catalog, plus fallback tail. + * + *

Examples (union member to effective rule): + *

    + *
  • {@code hdfs} in union: full hdfs catalog line (nn/dn/jn/hdfs principals map to hdfs)
    • + *
  • {@code nn} in union but catalog mapped name is hdfs: generated nn rule plus hdfs catalog line
    • + *
  • {@code foo} in union, not in catalog: generated foo/host@REALM to foo rule
    • + *
+ * Empty union falls back to {@link #composeFullCatalogRules()}. + */ + public String composeRulesForAllowedShortNames(Set allowedUserShortNames) { + if (allowedUserShortNames == null || allowedUserShortNames.isEmpty()) { + return composeFullCatalogRules(); + } + + Set normalizedAllowedShortNames = allowedUserShortNames.stream() + .filter(StringUtils::isNotBlank) + .map(String::trim) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (normalizedAllowedShortNames.isEmpty()) { + return composeFullCatalogRules(); + } + + List composedRuleLines = new ArrayList<>(); + Set shortNamesMatchedByCatalogRule = new LinkedHashSet<>(); + + for (PrimaryCatalogRule catalogRule : primaryCatalogRulesInOrder) { + if (catalogRule.mappedShortName != null && normalizedAllowedShortNames.contains(catalogRule.mappedShortName)) { + composedRuleLines.add(catalogRule.ruleLine); + shortNamesMatchedByCatalogRule.add(catalogRule.mappedShortName); + } + } + + List generatedSimpleRuleLines = normalizedAllowedShortNames.stream() + .filter(shortName -> !shortNamesMatchedByCatalogRule.contains(shortName)) + .sorted() + .map(AuthToLocalRuleCatalog::buildGeneratedShortNameRule) + .collect(Collectors.toList()); + composedRuleLines.addAll(generatedSimpleRuleLines); + composedRuleLines.addAll(fallbackRuleLines); + return joinRuleLines(composedRuleLines); + } + + public int getPrimaryCatalogRuleCount() { + return primaryCatalogRulesInOrder.size(); + } + + /** + * Mapped Kerberos short name from a catalog rule line (substitution suffix s/.star/<name>/). + * Example: hdfs catalog rule maps nn/dn/jn/hdfs principals to short name hdfs. + */ + public static String extractMappedShortName(String ruleLine) { + if (StringUtils.isBlank(ruleLine)) { + return null; + } + Matcher substitutionMatcher = RULE_SUBSTITUTION_SHORT_NAME_PATTERN.matcher(ruleLine); + if (substitutionMatcher.find()) { + return substitutionMatcher.group(1); + } + return null; + } + + private static List tokenizeRuleLines(String rawAuthToLocalRules) { + if (StringUtils.isBlank(rawAuthToLocalRules)) { + return Collections.emptyList(); + } + return Arrays.stream(rawAuthToLocalRules.split("\\s+")) + .map(String::trim) + .filter(ruleLine -> ruleLine.startsWith("RULE:") || "DEFAULT".equals(ruleLine)) + .collect(Collectors.toList()); + } + + private static boolean isFallbackKerberosRule(String ruleLine) { + return "DEFAULT".equals(ruleLine) || ruleLine.startsWith("RULE:[1:") || ruleLine.contains("s/@.*//"); + } + + /** Simple rule for allowlist short names absent from the XML catalog: {@code myapp/host@REALM -> myapp}. */ + private static String buildGeneratedShortNameRule(String shortName) { + return String.format(GENERATED_SHORT_NAME_RULE_TEMPLATE, shortName, shortName); + } + + private static String joinRuleLines(List ruleLines) { + return String.join("\n", ruleLines); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java new file mode 100644 index 0000000000..081bb6b790 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * Builds effective {@code auth_to_local} rules from the XML catalog and the global allowlist union + * ({@link #collectAllowedUserShortNames}). Rule text is not stored in the partition-plan Kafka JSON. + * + *

Two-step audit POST identity (see {@link ServiceAllowlistResolver}): + *

    + *
  1. Kerberos mapping — {@code KerberosName.getShortName()} uses composed rules (global union)
    2. + *
  2. Authorization — per-repo {@code services[repo].allowedUsers} (or static XML fallback)
    3. + *
+ */ +public class AuthToLocalRuleComposer { + private static final Logger LOG = LoggerFactory.getLogger(AuthToLocalRuleComposer.class); + + private static final AuthToLocalRuleComposer INSTANCE = new AuthToLocalRuleComposer(); + + private volatile AuthToLocalRuleCatalog ruleCatalog; + private volatile String lastAppliedRuleText; + private volatile int lastAppliedPartitionPlanVersion; + private volatile Boolean partitionPlanTopicExistsTestOverride; + + private AuthToLocalRuleComposer() { + } + + public static AuthToLocalRuleComposer getInstance() { + return INSTANCE; + } + + /** Loads the rule catalog from {@code ranger.audit.ingestor.auth.to.local} site XML. */ + public synchronized void initializeFromConfig() { + Properties ingestorProperties = AuditServerConfig.getInstance().getProperties(); + initializeFromProperties(ingestorProperties); + } + + public synchronized void initializeFromProperties(Properties ingestorProperties) { + String rawAuthToLocalRules = ingestorProperties.getProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL); + ruleCatalog = AuthToLocalRuleCatalog.parse(rawAuthToLocalRules); + lastAppliedRuleText = null; + lastAppliedPartitionPlanVersion = 0; + LOG.debug("Loaded auth_to_local catalog with {} primary rules", ruleCatalog.getPrimaryCatalogRuleCount()); + } + + /** Applies the full XML catalog (non-dynamic / startup fallback). */ + public void applyStaticRules() { + AuthToLocalRuleCatalog loadedRuleCatalog = requireRuleCatalog(); + applyKerberosNameRules(loadedRuleCatalog.composeFullCatalogRules(), 0); + } + + /** + * Dynamic-mode startup: when the partition-plan Kafka topic does not exist yet, apply the full XML + * catalog so Kerberos mapping works before {@link PartitionPlanWatcher} bootstraps the registry. + * When the topic already exists, defer to composed rules on {@link PartitionPlanHolder#install}. + */ + public void applyStartupRulesForDynamicMode(Properties ingestorProperties, String ingestorPropertyPrefix) { + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(ingestorProperties, ingestorPropertyPrefix)) { + return; + } + requireRuleCatalog(); + if (isPartitionPlanTopicPresent(ingestorProperties, ingestorPropertyPrefix)) { + LOG.info("Partition plan topic exists; auth_to_local rules will be composed from allowlisted short names on plan install"); + } else { + applyStaticRules(); + LOG.info("Partition plan topic does not exist yet; applied full auth_to_local catalog from XML until plan bootstrap"); + } + } + + /** + * When dynamic partition-plan mode is enabled, compose {@code auth_to_local} rules from the XML + * catalog plus the global allowlist union, then install via {@link KerberosName#setRules(String)}. + * + *

Called from {@link PartitionPlanHolder#install} on every plan version. Example plan: + * 

{@code
+     * services: {
+     *   dev_hdfs:  { allowedUsers: [hdfs, nn] },
+     *   dev_hive:  { allowedUsers: [hive] },
+     *   dev_foo:   { allowedUsers: [foo] }   // new plugin, not in XML catalog
+     * }
+     * }
+ * Union of hdfs, nn, hive, foo activates hdfs catalog rule (nn/dn/jn/hdfs principals to hdfs), + * hive catalog rule, and a generated foo/host@REALM to foo rule. Per-repo POST checks still + * use each repo's own allowlist, not the union. + */ + public void applyForPlan(PartitionPlan partitionPlan) { + Properties ingestorProperties = AuditServerConfig.getInstance().getProperties(); + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(ingestorProperties, PartitionPlanService.INGESTOR_PROP_PREFIX)) { + return; + } + if (partitionPlan == null) { + return; + } + + AuthToLocalRuleCatalog loadedRuleCatalog = requireRuleCatalog(); + Set allowedUserShortNames = collectAllowedUserShortNames(partitionPlan); + String composedRuleText = loadedRuleCatalog.composeRulesForAllowedShortNames(allowedUserShortNames); + applyKerberosNameRules(composedRuleText, partitionPlan.getVersion()); + LOG.info("Applied composed auth_to_local rules for plan version {} ({} active short names)", + partitionPlan.getVersion(), allowedUserShortNames.size()); + } + + /** Visible for tests — composes without applying Kerberos rules. */ + public String composeKerberosRulesForAllowedShortNames(Set allowedUserShortNames) { + return requireRuleCatalog().composeRulesForAllowedShortNames(allowedUserShortNames); + } + + /** Clears cached apply state between unit tests. */ + public synchronized void resetForTests() { + lastAppliedRuleText = null; + lastAppliedPartitionPlanVersion = 0; + partitionPlanTopicExistsTestOverride = null; + } + + /** When non-null, overrides {@link AuditMessageQueueUtils#partitionPlanTopicExists} for unit tests. */ + public void setPartitionPlanTopicExistsTestOverride(Boolean topicExists) { + partitionPlanTopicExistsTestOverride = topicExists; + } + + private boolean isPartitionPlanTopicPresent(Properties ingestorProperties, String ingestorPropertyPrefix) { + boolean ret = AuditMessageQueueUtils.partitionPlanTopicExists(ingestorProperties, ingestorPropertyPrefix); + Boolean topicExistsOverride = partitionPlanTopicExistsTestOverride; + + if (topicExistsOverride != null) { + ret = topicExistsOverride; + } + + return ret; + } + + /** + * Global allowlist union: all distinct {@code allowedUsers} short names across {@code plan.services}. + * Used only for {@code auth_to_local} composition (which mapping rules are active), not for POST authorization. + * + *

Example: dev_hdfs with hdfs and nn, dev_hive with hive, dev_ozone with om and ozone yields + * union hdfs, nn, hive, om, ozone. Repos not listed in services do not contribute. + */ + public static Set collectAllowedUserShortNames(PartitionPlan partitionPlan) { + Set allowedUserShortNames = new LinkedHashSet<>(); + if (partitionPlan == null || partitionPlan.getServices() == null) { + return allowedUserShortNames; + } + for (Map.Entry serviceEntry : partitionPlan.getServices().entrySet()) { + ServiceAllowlistEntry allowlistEntry = serviceEntry.getValue(); + if (allowlistEntry == null) { + continue; + } + for (String allowedUserShortName : allowlistEntry.getAllowedUsers()) { + if (StringUtils.isNotBlank(allowedUserShortName)) { + allowedUserShortNames.add(allowedUserShortName.trim()); + } + } + } + return allowedUserShortNames; + } + + private AuthToLocalRuleCatalog requireRuleCatalog() { + AuthToLocalRuleCatalog loadedRuleCatalog = ruleCatalog; + if (loadedRuleCatalog == null) { + initializeFromConfig(); + loadedRuleCatalog = ruleCatalog; + } + if (loadedRuleCatalog == null) { + throw new IllegalStateException("auth_to_local catalog is not loaded"); + } + return loadedRuleCatalog; + } + + private synchronized void applyKerberosNameRules(String composedRuleText, int partitionPlanVersion) { + if (StringUtils.isBlank(composedRuleText)) { + LOG.warn("Skipping auth_to_local apply: composed rules are blank"); + return; + } + if (composedRuleText.equals(lastAppliedRuleText) && partitionPlanVersion == lastAppliedPartitionPlanVersion) { + return; + } + try { + KerberosName.setRules(composedRuleText); + lastAppliedRuleText = composedRuleText; + lastAppliedPartitionPlanVersion = partitionPlanVersion; + LOG.debug("KerberosName auth_to_local rules updated (partitionPlanVersion={})", partitionPlanVersion); + } catch (Exception e) { + LOG.error("Failed to apply composed auth_to_local rules for plan version {}: {}", + partitionPlanVersion, e.getMessage(), e); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java new file mode 100644 index 0000000000..029acc813c --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; + +import java.util.Properties; + +/** Grows the audit topic partition count before a plan references new tail IDs. */ +public class KafkaAuditTopicPartitionGrower { + /** Increases {@code ranger_audits} partition count when the plan needs more lanes. */ + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + AuditMessageQueueUtils.ensureTopicPartitionCount(props, propPrefix, auditTopicName, requiredPartitionCount); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java new file mode 100644 index 0000000000..198219ae5d --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; + +/** Kafka compacted topic implementation of {@link PartitionPlanRegistry}. */ +public class KafkaPartitionPlanRegistry implements PartitionPlanRegistry { + private static final Logger LOG = LoggerFactory.getLogger(KafkaPartitionPlanRegistry.class); + + private final Properties props; + private final String propPrefix; + private final String planTopic; + private final int consumerPollTimeoutMs; + private final KafkaProducer producer; + + public KafkaPartitionPlanRegistry(Properties props, String propPrefix) throws Exception { + this.props = props; + this.propPrefix = propPrefix; + this.planTopic = PartitionPlanKafkaConfig.resolvePlanTopicName(props, propPrefix); + this.consumerPollTimeoutMs = PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, propPrefix); + AuditMessageQueueUtils.createPartitionPlanTopicIfNotExists(props, propPrefix); + this.producer = new KafkaProducer<>(PartitionPlanKafkaConfig.producerConfig(props, propPrefix)); + LOG.info("Kafka partition plan registry ready for topic '{}'", planTopic); + } + + /** Reads the latest compacted value for the audit topic key from partition 0. */ + @Override + public PartitionPlan readPlan(String auditTopicKey) { + requireAuditTopicKey(auditTopicKey); + try (KafkaConsumer consumer = openConsumer()) { + return readLatestCompactedPlan(consumer, auditTopicKey); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + LOG.error("Failed to read partition plan from Kafka topic '{}' for audit topic key '{}'", planTopic, auditTopicKey, e); + throw new PartitionPlanException("Failed to read partition plan from Kafka topic '" + planTopic + "'", e); + } + } + + /** Publishes a new plan version to the compacted topic (key = audit topic name). */ + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + requireAuditTopicKey(auditTopicKey); + if (plan == null) { + throw new PartitionPlanException("Partition plan is required"); + } + PartitionPlanValidator.validate(plan); + try { + producer.send(new ProducerRecord<>(planTopic, auditTopicKey, plan.toJson())).get(); + LOG.info("Wrote partition plan version {} for audit topic '{}' to '{}'", plan.getVersion(), auditTopicKey, planTopic); + } catch (Exception e) { + LOG.error("Failed to write partition plan version {} to Kafka topic '{}' for audit topic key '{}'", plan.getVersion(), planTopic, auditTopicKey, e); + throw new PartitionPlanException("Failed to write partition plan to Kafka topic '" + planTopic + "'", e); + } + } + + @Override + public void close() { + producer.flush(); + producer.close(); + } + + private KafkaConsumer openConsumer() throws Exception { + return new KafkaConsumer<>(PartitionPlanKafkaConfig.consumerConfig(props, propPrefix, PartitionPlanConstants.PLAN_REGISTRY_CONSUMER_GROUP)); + } + + private PartitionPlan readLatestCompactedPlan(KafkaConsumer consumer, String auditTopicKey) { + TopicPartition partition = new TopicPartition(planTopic, 0); + consumer.assign(Collections.singletonList(partition)); + consumer.seekToBeginning(Collections.singletonList(partition)); + + PartitionPlan latest = null; + ConsumerRecords records; + do { + records = consumer.poll(Duration.ofMillis(consumerPollTimeoutMs)); + for (ConsumerRecord record : records) { + if (auditTopicKey.equals(record.key())) { + latest = PartitionPlan.fromJson(record.value()); + } + } + } while (!records.isEmpty()); + return latest; + } + + private static void requireAuditTopicKey(String auditTopicKey) { + if (StringUtils.isBlank(auditTopicKey)) { + throw new PartitionPlanException("auditTopicKey is required"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java new file mode 100644 index 0000000000..07104017a6 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +/** Append-only plan updates: promote unknown plugins and scale hot plugins without reshuffling. */ +public class PartitionPlanAllocator { + private PartitionPlanAllocator() { + } + + /** + * Onboard a plugin: promote from buffer and register service allowlists tagged with {@code pluginId}. + */ + public static PartitionPlan onboardPlugin(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap, String updatedBy) { + requireMutationInputs(current, pluginId, partitionCount, updatedBy); + if (current.getPlugins().containsKey(pluginId)) { + assertOnboardNotConflicting(current, pluginId, partitionCount, servicesMap); + throw new PartitionPlanException("Plugin '" + pluginId + "' already has dedicated partitions"); + } + + List remainingBuffer = new ArrayList<>(current.getBuffer().getPartitions()); + List newPluginIds = takeFromBuffer(remainingBuffer, partitionCount); + int topicPartitionCount = appendTailPartitions(newPluginIds, current.getTopicPartitionCount(), partitionCount - newPluginIds.size()); + + Map plugins = addPluginAssignment(current, pluginId, newPluginIds); + Map services = mergeServicesWithPluginId(current.getServices(), servicesMap, pluginId); + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, remainingBuffer, services); + } + + /** + * Update an onboarded plugin: scale tail partitions and/or mutate service allowlists in one version bump. + */ + public static PartitionPlan updatePlugin(PartitionPlan current, String pluginId, UpdatePlugin updateRequest, String updatedBy) { + if (current == null || updateRequest == null) { + throw new PartitionPlanException("Current plan and update request are required"); + } + if (StringUtils.isBlank(pluginId) || StringUtils.isBlank(updatedBy)) { + throw new PartitionPlanException("pluginId and updatedBy are required"); + } + PartitionPlanValidator.validate(current); + + Map services = new LinkedHashMap<>(current.getServices()); + Map plugins = new LinkedHashMap<>(current.getPlugins()); + List bufferIds = new ArrayList<>(current.getBuffer().getPartitions()); + int topicPartitionCount = current.getTopicPartitionCount(); + + applyServiceRemovals(current, services, pluginId, updateRequest.getRemoveServices()); + applyServiceUpdates(current, services, pluginId, updateRequest.getUpdateServices()); + applyServiceAdditions(current, services, pluginId, updateRequest.getAddServices()); + + Integer additionalPartitions = updateRequest.getAdditionalPartitions(); + if (additionalPartitions != null && additionalPartitions >= 1) { + if (!plugins.containsKey(pluginId)) { + throw new PartitionPlanException("Plugin '" + pluginId + "' is not configured; onboard it first"); + } + List pluginIds = new ArrayList<>(plugins.get(pluginId).getPartitions()); + topicPartitionCount = appendTailPartitions(pluginIds, topicPartitionCount, additionalPartitions); + plugins.put(pluginId, new PluginPartitionAssignment(pluginIds)); + } + + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, bufferIds, services); + } + + public static PartitionPlan promotePlugin(PartitionPlan current, String pluginId, int partitionCount, String updatedBy) { + return promotePlugin(current, pluginId, partitionCount, updatedBy, null, null); + } + + /** + * Give a plugin its own partitions. Uses buffer IDs first; adds new tail IDs when buffer is too small. + * Optionally upserts {@code services[repo]} in the same plan version when {@code repo} and {@code allowedUsers} are set. + */ + public static PartitionPlan promotePlugin(PartitionPlan current, String pluginId, int partitionCount, String updatedBy, String repo, List allowedUsers) { + requireMutationInputs(current, pluginId, partitionCount, updatedBy); + if (current.getPlugins().containsKey(pluginId)) { + assertPromoteNotConflicting(current, pluginId, partitionCount, repo, allowedUsers); + throw new PartitionPlanException("Plugin '" + pluginId + "' already has dedicated partitions"); + } + + List remainingBuffer = new ArrayList<>(current.getBuffer().getPartitions()); + List newPluginIds = takeFromBuffer(remainingBuffer, partitionCount); + int topicPartitionCount = appendTailPartitions(newPluginIds, current.getTopicPartitionCount(), partitionCount - newPluginIds.size()); + + Map plugins = addPluginAssignment(current, pluginId, newPluginIds); + Map services = mergeServiceAllowlist(current.getServices(), repo, allowedUsers); + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, remainingBuffer, services); + } + + /** + * Onboard a Ranger service repo: promote plugin partitions and upsert service allowlist atomically. + */ + public static PartitionPlan onboardRepo(PartitionPlan current, String repo, String pluginId, int partitionCount, List allowedUsers, String updatedBy) { + if (StringUtils.isBlank(repo)) { + throw new PartitionPlanException("repo is required"); + } + if (allowedUsers == null || allowedUsers.isEmpty()) { + throw new PartitionPlanException("allowedUsers are required"); + } + return promotePlugin(current, pluginId, partitionCount, updatedBy, repo, allowedUsers); + } + + /** + * Add more partitions to an existing plugin by appending new tail IDs only. + */ + public static PartitionPlan scalePlugin(PartitionPlan current, String pluginId, int additionalPartitions, String updatedBy) { + requireMutationInputs(current, pluginId, additionalPartitions, updatedBy); + if (!current.getPlugins().containsKey(pluginId)) { + throw new PartitionPlanException("Plugin '" + pluginId + "' is not configured; promote it first"); + } + + List pluginIds = new ArrayList<>(current.getPlugins().get(pluginId).getPartitions()); + int topicPartitionCount = appendTailPartitions(pluginIds, current.getTopicPartitionCount(), additionalPartitions); + + return commitPlanUpdate(current, updatedBy, topicPartitionCount, addPluginAssignment(current, pluginId, pluginIds), current.getBuffer().getPartitions(), current.getServices()); + } + + /** + * True when plugin onboard with {@code partitionCount} and optional services already matches the current plan. + */ + public static boolean isOnboardAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap) { + boolean ret = false; + + if (current != null && pluginHasPartitionCount(current, pluginId, partitionCount)) { + ret = true; + + if (servicesMap != null && !servicesMap.isEmpty()) { + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry existing = current.getServices().get(repo); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } + } + } + + return ret; + } + + /** + * True when a service-only update request is already satisfied (scale mutations are never treated as no-op). + */ + public static boolean isUpdateAlreadyApplied(PartitionPlan current, String pluginId, UpdatePlugin updateRequest) { + boolean ret = false; + + if (current != null && updateRequest != null) { + Integer additionalPartitions = updateRequest.getAdditionalPartitions(); + + if (additionalPartitions == null || additionalPartitions < 1) { + ret = true; + + for (String repo : updateRequest.getRemoveServices()) { + if (current.getServices().containsKey(repo.trim())) { + ret = false; + + break; + } + } + + if (ret) { + for (Map.Entry entry : updateRequest.getAddServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } + } + + if (ret) { + for (Map.Entry entry : updateRequest.getUpdateServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } + } + + if (ret) { + ret = updateRequest.hasMutationDelta(); + } + } + } + + return ret; + } + + /** + * True when the plugin is already promoted with {@code partitionCount} slots and, when {@code repo} is set, + * the service allowlist already matches {@code allowedUsers}. + */ + public static boolean isPromoteAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { + boolean ret = false; + + if (current != null && pluginHasPartitionCount(current, pluginId, partitionCount)) { + if (StringUtils.isNotBlank(repo)) { + ServiceAllowlistEntry existing = current.getServices().get(repo.trim()); + ret = existing != null && existing.hasSameAllowedUsers(allowedUsers); + } else { + ret = true; + } + } + + return ret; + } + + /** Applies a merged plan with append-only checks against the current plan. */ + public static PartitionPlan replacePlan(PartitionPlan current, PartitionPlan proposed) { + if (current == null || proposed == null) { + throw new PartitionPlanException("Current and proposed plans are required"); + } + if (!StringUtils.equals(current.getTopic(), proposed.getTopic())) { + throw new PartitionPlanException("Proposed topic must match current topic"); + } + PartitionPlan next = proposed.toBuilder().version(current.getVersion() + 1).build(); + PartitionPlanValidator.validate(next); + PartitionPlanValidator.validateAppendOnly(current, next); + return next; + } + + private static void applyServiceRemovals(PartitionPlan current, Map services, String pluginId, List removeServices) { + for (String repoName : removeServices) { + String repo = repoName.trim(); + verifyServiceOwnedByPlugin(current, repo, pluginId); + services.remove(repo); + } + } + + private static void applyServiceUpdates(PartitionPlan current, Map services, String pluginId, Map updateServices) { + for (Map.Entry entry : updateServices.entrySet()) { + String repo = entry.getKey().trim(); + verifyServiceOwnedByPlugin(current, repo, pluginId); + services.put(repo, withPluginId(entry.getValue(), pluginId)); + } + } + + private static void applyServiceAdditions(PartitionPlan current, Map services, String pluginId, Map addServices) { + for (Map.Entry entry : addServices.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry existing = current.getServices().get(repo); + if (existing != null) { + assertServiceOwnedByPlugin(existing, repo, pluginId); + } + services.put(repo, withPluginId(entry.getValue(), pluginId)); + } + } + + private static void verifyServiceOwnedByPlugin(PartitionPlan current, String repo, String pluginId) { + ServiceAllowlistEntry existing = current.getServices().get(repo); + if (existing == null) { + throw new PartitionPlanException("Service '" + repo + "' is not configured"); + } + assertServiceOwnedByPlugin(existing, repo, pluginId); + } + + private static void assertServiceOwnedByPlugin(ServiceAllowlistEntry existing, String repo, String pluginId) { + if (existing.getPluginId() != null && !Objects.equals(existing.getPluginId(), pluginId)) { + throw new PartitionPlanException("Service '" + repo + "' belongs to plugin '" + existing.getPluginId() + "'"); + } + } + + private static boolean serviceEntryMatches(ServiceAllowlistEntry existing, ServiceAllowlistEntry expected, String pluginId) { + return existing.hasSameAllowedUsers(expected.getAllowedUsers()) && Objects.equals(existing.getPluginId(), pluginId); + } + + private static ServiceAllowlistEntry withPluginId(ServiceAllowlistEntry entry, String pluginId) { + if (entry == null) { + throw new PartitionPlanException("Service allowlist entry is required"); + } + return new ServiceAllowlistEntry(entry.getAllowedUsers(), entry.getSource(), entry.getNotes(), pluginId); + } + + /** Pull up to count partition IDs from the front of the buffer list. */ + private static List takeFromBuffer(List bufferIds, int count) { + List taken = new ArrayList<>(Math.min(count, bufferIds.size())); + while (taken.size() < count && !bufferIds.isEmpty()) { + taken.add(bufferIds.remove(0)); + } + return taken; + } + + /** Append new tail partition IDs and return the new topic partition count. */ + private static int appendTailPartitions(List target, int topicPartitionCount, int count) { + for (int i = 0; i < count; i++) { + target.add(topicPartitionCount++); + } + return topicPartitionCount; + } + + private static Map addPluginAssignment(PartitionPlan current, String pluginId, List partitionIds) { + Map plugins = new LinkedHashMap<>(current.getPlugins()); + plugins.put(pluginId, new PluginPartitionAssignment(partitionIds)); + return plugins; + } + + private static PartitionPlan commitPlanUpdate(PartitionPlan current, String updatedBy, int topicPartitionCount, Map plugins, List bufferIds, Map services) { + PartitionPlan next = current.toBuilder() + .version(current.getVersion() + 1) + .topicPartitionCount(topicPartitionCount) + .plugins(plugins) + .buffer(new PluginPartitionAssignment(bufferIds)) + .services(services != null ? services : current.getServices()) + .updatedAt(Instant.now().toString()) + .updatedBy(updatedBy) + .build(); + PartitionPlanValidator.validate(next); + PartitionPlanValidator.validateAppendOnly(current, next); + return next; + } + + private static Map mergeServicesWithPluginId(Map currentServices, Map servicesMap, String pluginId) { + Map services = new LinkedHashMap<>(currentServices); + if (servicesMap == null || servicesMap.isEmpty()) { + return services; + } + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry tagged = withPluginId(entry.getValue(), pluginId); + ServiceAllowlistEntry existing = services.get(repo); + if (existing != null && !existing.hasSameAllowedUsers(tagged.getAllowedUsers())) { + throw new PartitionPlanException("Service '" + repo + "' already exists with different allowedUsers"); + } + if (existing != null && existing.getPluginId() != null && !Objects.equals(existing.getPluginId(), pluginId)) { + throw new PartitionPlanException("Service '" + repo + "' belongs to plugin '" + existing.getPluginId() + "'"); + } + services.put(repo, tagged); + } + return services; + } + + private static Map mergeServiceAllowlist(Map currentServices, String repo, List allowedUsers) { + Map services = new LinkedHashMap<>(currentServices); + if (StringUtils.isNotBlank(repo) && allowedUsers != null && !allowedUsers.isEmpty()) { + services.put(repo.trim(), ServiceAllowlistEntry.ofUsers(allowedUsers)); + } + return services; + } + + private static boolean pluginHasPartitionCount(PartitionPlan current, String pluginId, int partitionCount) { + PluginPartitionAssignment assignment = current.getPlugins().get(pluginId); + return assignment != null && assignment.getPartitions().size() == partitionCount; + } + + private static void assertOnboardNotConflicting(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap) { + PluginPartitionAssignment existing = requireNonNull(current.getPlugins().get(pluginId)); + if (existing.getPartitions().size() != partitionCount) { + throw new PartitionPlanException("Plugin '" + pluginId + "' already has " + existing.getPartitions().size() + " dedicated partition(s); requested " + partitionCount); + } + if (servicesMap != null) { + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry serviceEntry = current.getServices().get(repo); + if (serviceEntry != null && !serviceEntry.hasSameAllowedUsers(entry.getValue().getAllowedUsers())) { + throw new PartitionPlanException("Service '" + repo + "' already exists with different allowedUsers"); + } + } + } + } + + private static void assertPromoteNotConflicting(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { + PluginPartitionAssignment existing = requireNonNull(current.getPlugins().get(pluginId)); + if (existing.getPartitions().size() != partitionCount) { + throw new PartitionPlanException("Plugin '" + pluginId + "' already has " + existing.getPartitions().size() + " dedicated partition(s); requested " + partitionCount); + } + if (StringUtils.isNotBlank(repo)) { + ServiceAllowlistEntry serviceEntry = current.getServices().get(repo.trim()); + if (serviceEntry != null && !serviceEntry.hasSameAllowedUsers(allowedUsers)) { + throw new PartitionPlanException("Service '" + repo.trim() + "' already exists with different allowedUsers"); + } + } + } + + private static void requireMutationInputs(PartitionPlan current, String pluginId, int partitionCount, String updatedBy) { + if (current == null) { + throw new PartitionPlanException("Current plan is required"); + } + PartitionPlanValidator.validate(current); + if (StringUtils.isBlank(pluginId) || partitionCount < 1 || StringUtils.isBlank(updatedBy)) { + throw new PartitionPlanException("pluginId, partitionCount, and updatedBy are required"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java new file mode 100644 index 0000000000..7ab09b8e8d --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Builds the initial bootstrap plan from legacy XML and seeds the registry when the plan topic is empty. */ +public class PartitionPlanBootstrap { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanBootstrap.class); + + private PartitionPlanBootstrap() { + } + + /** Builds the initial bootstrap plan ({@link PartitionPlanConstants#INITIAL_PLAN_VERSION}) using the same contiguous layout as static {@code AuditPartitioner}. */ + public static PartitionPlan createInitialPlan(PartitionPlanBootstrapConfig config) { + if (config == null || StringUtils.isBlank(config.getAuditTopic())) { + throw new PartitionPlanException("Audit topic and bootstrap config are required"); + } + + Map plugins = new LinkedHashMap<>(); + int nextPartition = 0; + for (String plugin : config.getConfiguredPlugins()) { + if (StringUtils.isBlank(plugin)) { + continue; + } + int count = config.getPartitionsForPlugin(plugin.trim()); + plugins.put(plugin.trim(), PluginPartitionAssignment.ofRange(nextPartition, nextPartition + count - 1)); + nextPartition += count; + } + + int topicPartitionCount; + if (nextPartition == 0) { + topicPartitionCount = config.getHashBasedTopicPartitionCount(); + } else { + topicPartitionCount = nextPartition + Math.max(1, config.getBufferPartitionCount()); + } + PartitionPlan plan = PartitionPlan.builder() + .topic(config.getAuditTopic()) + .version(PartitionPlanConstants.INITIAL_PLAN_VERSION) + .topicPartitionCount(topicPartitionCount) + .updatedAt(Instant.now().toString()) + .updatedBy(PartitionPlanConstants.BOOTSTRAP_UPDATED_BY) + .plugins(plugins) + .buffer(PluginPartitionAssignment.ofRange(nextPartition, topicPartitionCount - 1)) + .services(ServiceAllowlistBootstrap.loadAllowlistsFromServerConfig()) + .build(); + + PartitionPlanValidator.validate(plan); + return plan; + } + + /** Builds the first plan from legacy ingestor producer/XML configuration. */ + public static PartitionPlan createInitialPlanFromProducerConfig(Map producerConfig, String auditTopic) { + return createInitialPlan(PartitionPlanBootstrapConfig.fromProducerConfigMap(producerConfig, auditTopic)); + } + + /** + * Empty-registry bootstrap: read registry, build the initial bootstrap plan from XML when no plan + * exists, re-read before publish (concurrent pods), write once, then mandatory read-back. + */ + public static PartitionPlan bootstrapIfEmpty(PartitionPlanRegistry registry, String auditTopic, Map producerConfig) { + PartitionPlan plan = registry.readPlan(auditTopic); + if (plan != null) { + LOG.info("Partition plan version {} already present for audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } + + PartitionPlan localPlan = createInitialPlanFromProducerConfig(producerConfig, auditTopic); + plan = registry.readPlan(auditTopic); + if (plan != null) { + LOG.info("Peer published partition plan version {} while bootstrapping audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } + + registry.writePlan(auditTopic, localPlan); + plan = registry.readPlan(auditTopic); + if (plan == null) { + LOG.error("Mandatory read-back failed after publishing initial bootstrap plan for audit topic '{}'", auditTopic); + throw new PartitionPlanException("Mandatory read-back failed after publishing bootstrap plan for audit topic '" + auditTopic + "'"); + } + LOG.info("Bootstrap partition plan version {} published and read back for audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java new file mode 100644 index 0000000000..349cf5deff --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.server.AuditServerConstants; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Inputs for building the first partition plan from legacy ingestor XML / producer config. */ +public class PartitionPlanBootstrapConfig { + private final String auditTopic; + private final String[] configuredPlugins; + private final int defaultPartitionsPerPlugin; + private final int bufferPartitionCount; + private final int hashBasedTopicPartitionCount; + private final Map pluginPartitionOverrides; + + public PartitionPlanBootstrapConfig(String auditTopic, String[] configuredPlugins, int defaultPartitionsPerPlugin, int bufferPartitionCount, int hashBasedTopicPartitionCount, Map pluginPartitionOverrides) { + this.auditTopic = auditTopic; + this.configuredPlugins = configuredPlugins != null ? configuredPlugins : new String[0]; + this.defaultPartitionsPerPlugin = defaultPartitionsPerPlugin; + this.bufferPartitionCount = bufferPartitionCount; + this.hashBasedTopicPartitionCount = Math.max(1, hashBasedTopicPartitionCount); + this.pluginPartitionOverrides = pluginPartitionOverrides != null ? new LinkedHashMap<>(pluginPartitionOverrides) : Collections.emptyMap(); + } + + public static PartitionPlanBootstrapConfig create(String auditTopic, String[] configuredPlugins, int defaultPartitionsPerPlugin, int bufferPartitionCount) { + return new PartitionPlanBootstrapConfig(auditTopic, configuredPlugins, defaultPartitionsPerPlugin, bufferPartitionCount, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS, Collections.emptyMap()); + } + + public PartitionPlanBootstrapConfig withPluginOverride(String pluginId, int partitionCount) { + Map overrides = new LinkedHashMap<>(pluginPartitionOverrides); + overrides.put(pluginId, partitionCount); + return new PartitionPlanBootstrapConfig(auditTopic, configuredPlugins, defaultPartitionsPerPlugin, bufferPartitionCount, hashBasedTopicPartitionCount, overrides); + } + + public String getAuditTopic() { + return auditTopic; + } + + public String[] getConfiguredPlugins() { + return configuredPlugins; + } + + public int getBufferPartitionCount() { + return bufferPartitionCount; + } + + /** Used when {@link #getConfiguredPlugins()} is empty: matches {@code kafka.topic.partitions} / hash-based mode. */ + public int getHashBasedTopicPartitionCount() { + return hashBasedTopicPartitionCount; + } + + public int getPartitionsForPlugin(String pluginId) { + Integer override = pluginPartitionOverrides.get(pluginId); + int count = override != null ? override : defaultPartitionsPerPlugin; + return Math.max(1, count); + } + + public static PartitionPlanBootstrapConfig fromProducerConfigMap(Map configs, String auditTopic) { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + String pluginsStr = getString(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); + String[] plugins = parsePluginIds(pluginsStr); + int defaultPerPlugin = getInt(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + int bufferCount = getInt(configs, propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, AuditServerConstants.DEFAULT_BUFFER_PARTITIONS); + int hashBasedTopicPartitions = getInt(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS); + + PartitionPlanBootstrapConfig config = new PartitionPlanBootstrapConfig(auditTopic, plugins, defaultPerPlugin, bufferCount, hashBasedTopicPartitions, Collections.emptyMap()); + for (String plugin : plugins) { + String overrideKey = propPrefix + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin; + if (configs.containsKey(overrideKey)) { + config = config.withPluginOverride(plugin, getInt(configs, overrideKey, defaultPerPlugin)); + } + } + return config; + } + + private static String[] parsePluginIds(String pluginsStr) { + if (pluginsStr == null || pluginsStr.isBlank()) { + return new String[0]; + } + List plugins = new ArrayList<>(); + for (String plugin : pluginsStr.split(",")) { + if (plugin != null && !plugin.isBlank()) { + plugins.add(plugin.trim()); + } + } + return plugins.toArray(new String[0]); + } + + private static String getString(Map configs, String key, String defaultValue) { + Object val = configs.get(key); + return val != null ? val.toString() : defaultValue; + } + + private static int getInt(Map configs, String key, int defaultValue) { + Object val = configs.get(key); + if (val == null) { + return defaultValue; + } + if (val instanceof Number) { + return ((Number) val).intValue(); + } + try { + return Integer.parseInt(val.toString()); + } catch (NumberFormatException e) { + return defaultValue; + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java new file mode 100644 index 0000000000..325d838693 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** Hot-path in-memory plan for {@code AuditPartitioner} and the background watcher. */ +public class PartitionPlanHolder { + private static final PartitionPlanHolder INSTANCE = new PartitionPlanHolder(); + + private final AtomicReference planRef = new AtomicReference<>(); + private volatile int lastInstalledVersion; + + private PartitionPlanHolder() { + } + + public static PartitionPlanHolder getInstance() { + return INSTANCE; + } + + public PartitionPlan getPlan() { + return planRef.get(); + } + + public int getLastInstalledVersion() { + return lastInstalledVersion; + } + + /** Validates and atomically installs the plan used by the Kafka partitioner. */ + public void install(PartitionPlan plan, Integer kafkaPartitionCount) { + PartitionPlanValidator.validate(plan, kafkaPartitionCount); + planRef.set(plan); + lastInstalledVersion = plan.getVersion(); + AuthToLocalRuleComposer.getInstance().applyForPlan(plan); + } + + /** + * Returns allowed short usernames for a service repo from the in-memory registry document. + * {@code null} when the plan has no {@code services} block, or when the repo is not present + * in the plan (caller should fall back to static XML). + * Returns a non-empty set when the repo is present; {@link PartitionPlanValidator} rejects + * plans whose {@code allowedUsers} list is empty at install time. + * + *

Used by {@link ServiceAllowlistResolver} for per-repo POST authorization — not for the global + * allowlist union ({@link AuthToLocalRuleComposer#collectAllowedUserShortNames}). + */ + public Set getAllowedUsersForService(String serviceName) { + Set ret = null; + PartitionPlan plan = planRef.get(); + + if (plan != null && plan.getServices() != null && !plan.getServices().isEmpty()) { + ServiceAllowlistEntry entry = plan.getServices().get(serviceName); + + if (entry != null) { + if (entry.getAllowedUsers().isEmpty()) { + ret = Collections.emptySet(); + } else { + ret = Collections.unmodifiableSet(new LinkedHashSet<>(entry.getAllowedUsers())); + } + } + } + + return ret; + } + + /** Clears holder state between unit tests. */ + public void resetForTests() { + planRef.set(null); + lastInstalledVersion = 0; + AuthToLocalRuleComposer.getInstance().resetForTests(); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java new file mode 100644 index 0000000000..8c6275206b --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** Kafka client settings for the partition-plan registry topic. */ +public class PartitionPlanKafkaConfig { + private PartitionPlanKafkaConfig() { + } + + /** Resolves the compacted Kafka topic that stores the partition plan registry. */ + public static String resolvePlanTopicName(Properties props, String propPrefix) { + return MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + } + + /** Returns whether ingestor should load routing from the Kafka plan registry. */ + public static boolean isDynamicPartitionPlanEnabled(Properties props, String propPrefix) { + return MiscUtil.getBooleanProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, false); + } + + /** Resolves short usernames allowed to call partition-plan admin REST (empty = any authenticated principal). */ + public static Set resolvePartitionPlanAdminUsers(Properties props, String propPrefix) { + Set ret = Collections.emptySet(); + String configured = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_ALLOWED_USERS, ""); + + if (configured != null && !configured.isBlank()) { + Set users = new LinkedHashSet<>(); + + for (String user : configured.split(",")) { + if (user != null && !user.isBlank()) { + users.add(user.trim()); + } + } + ret = users; + } + + return ret; + } + + /** Returns whether the Kafka producer partitioner should use the in-memory dynamic plan. */ + public static boolean isDynamicPartitionPlanEnabled(Map configs, String ingestorPropPrefix) { + boolean ret = false; + String key = ingestorPropPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED; + Object val = configs.get(key); + + if (val != null) { + if (val instanceof Boolean) { + ret = (Boolean) val; + } else { + ret = Boolean.parseBoolean(val.toString()); + } + } + + return ret; + } + + /** Resolves how often each ingestor pod reloads the plan from Kafka. */ + public static int resolveRefreshIntervalMs(Properties props, String propPrefix) { + return MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_REFRESH_INTERVAL_MS, AuditServerConstants.DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS); + } + + /** Resolves the Kafka consumer poll timeout when draining the compacted plan topic. */ + public static int resolveConsumerPollTimeoutMs(Properties props, String propPrefix) { + return MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, AuditServerConstants.DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS); + } + + /** Builds Kafka producer properties for writing to the plan registry topic. */ + public static Properties producerConfig(Properties props, String propPrefix) throws Exception { + Properties producerProps = new Properties(); + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); + producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, AuditServerConstants.DEFAULT_PRODUCER_REQUEST_TIMEOUT_MS)); + applySecurity(producerProps, props, propPrefix); + return producerProps; + } + + /** Builds Kafka consumer properties for reading the plan registry topic. */ + public static Properties consumerConfig(Properties props, String propPrefix, String groupId) throws Exception { + Properties consumerProps = new Properties(); + consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, AuditServerConstants.DEFAULT_PRODUCER_REQUEST_TIMEOUT_MS)); + applySecurity(consumerProps, props, propPrefix); + return consumerProps; + } + + /** Applies Kerberos/SASL settings shared by plan-registry Kafka clients. */ + private static void applySecurity(Properties clientProps, Properties props, String propPrefix) throws Exception { + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + clientProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + clientProps.put(AuditServerConstants.PROP_SASL_MECHANISM, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM)); + clientProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) { + clientProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, AuditMessageQueueUtils.getJAASConfig(props, propPrefix)); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java new file mode 100644 index 0000000000..4d2c190b43 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; + +/** Durable store for partition plans (Kafka compacted topic in dynamic mode). */ +public interface PartitionPlanRegistry extends AutoCloseable { + PartitionPlan readPlan(String auditTopicKey); + + void writePlan(String auditTopicKey, PartitionPlan plan); + + @Override + void close(); +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java new file mode 100644 index 0000000000..2b07cdc3dc --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import java.util.Properties; + +/** Opens a Kafka-backed {@link PartitionPlanRegistry} for REST mutations. */ +public class PartitionPlanRegistryFactory { + /** Creates a registry connected to the compacted plan topic. */ + public PartitionPlanRegistry open(Properties props, String propPrefix) throws Exception { + return new KafkaPartitionPlanRegistry(props, propPrefix); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java new file mode 100644 index 0000000000..a1ea647535 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; + +import java.util.List; +import java.util.Map; + +/** Validates partition-plan REST mutation request bodies before registry writes. */ +public class PartitionPlanRequestValidator { + private PartitionPlanRequestValidator() { + } + + public static void validateOnboardPlugin(OnboardPlugin onboardPluginRequest) { + if (onboardPluginRequest == null) { + throw new PartitionPlanException("Onboard plugin request is required"); + } + validateNonBlankPluginId(onboardPluginRequest.getPluginId()); + validatePositiveCount(onboardPluginRequest.getPartitionCount(), "partitionCount"); + validateExpectedVersion(onboardPluginRequest.getExpectedVersion()); + validateRequiredServiceEntries(onboardPluginRequest.getServices()); + } + + public static void validateUpdatePlugin(String pluginId, UpdatePlugin updatePluginRequest) { + if (updatePluginRequest == null) { + throw new PartitionPlanException("Update plugin request is required"); + } + validateNonBlankPluginId(pluginId); + validateExpectedVersion(updatePluginRequest.getExpectedVersion()); + if (!updatePluginRequest.hasMutationDelta()) { + throw new PartitionPlanException("At least one of additionalPartitions, addServices, updateServices, or removeServices must be provided"); + } + Integer additionalPartitions = updatePluginRequest.getAdditionalPartitions(); + if (additionalPartitions != null && additionalPartitions < 1) { + throw new PartitionPlanException("additionalPartitions must be >= 1"); + } + validateOptionalServiceEntries(updatePluginRequest.getAddServices()); + validateOptionalServiceEntries(updatePluginRequest.getUpdateServices()); + validateRemoveServiceNames(updatePluginRequest.getRemoveServices()); + } + + private static void validateRequiredServiceEntries(Map services) { + if (services == null || services.isEmpty()) { + throw new PartitionPlanException("services are required"); + } + validateServiceEntries(services); + } + + private static void validateOptionalServiceEntries(Map services) { + if (services == null || services.isEmpty()) { + return; + } + validateServiceEntries(services); + } + + private static void validateServiceEntries(Map services) { + for (Map.Entry entry : services.entrySet()) { + validateNonBlankServiceName(entry.getKey()); + if (entry.getValue() == null) { + throw new PartitionPlanException("Service allowlist entry is required for '" + entry.getKey() + "'"); + } + validateNonEmptyAllowedUsers(entry.getValue().getAllowedUsers()); + } + } + + private static void validateRemoveServiceNames(List removeServices) { + if (removeServices == null || removeServices.isEmpty()) { + return; + } + for (String serviceName : removeServices) { + validateNonBlankServiceName(serviceName); + } + } + + private static void validateExpectedVersion(int expectedVersion) { + if (expectedVersion < PartitionPlanConstants.INITIAL_PLAN_VERSION) { + throw new PartitionPlanException("expectedVersion must be >= " + PartitionPlanConstants.INITIAL_PLAN_VERSION); + } + } + + private static void validateNonBlankPluginId(String pluginId) { + if (StringUtils.isBlank(pluginId)) { + throw new PartitionPlanException("pluginId is required"); + } + } + + private static void validateNonBlankServiceName(String serviceName) { + if (StringUtils.isBlank(serviceName)) { + throw new PartitionPlanException("serviceName is required"); + } + } + + private static void validatePositiveCount(int count, String fieldName) { + if (count < 1) { + throw new PartitionPlanException(fieldName + " must be >= 1"); + } + } + + private static void validateNonEmptyAllowedUsers(List allowedUsers) { + if (allowedUsers == null || allowedUsers.isEmpty()) { + throw new PartitionPlanException("allowedUsers are required"); + } + + boolean hasNonBlankUser = false; + + for (String allowedUserShortName : allowedUsers) { + if (StringUtils.isNotBlank(allowedUserShortName)) { + hasNonBlankUser = true; + + break; + } + } + + if (!hasNonBlankUser) { + throw new PartitionPlanException("allowedUsers must contain at least one non-blank username"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java new file mode 100644 index 0000000000..ce4d3c8651 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.springframework.stereotype.Component; + +import java.util.Collections; +import java.util.Properties; +import java.util.Set; + +/** REST mutations and reads for the dynamic Kafka partition plan. */ +@Component +public class PartitionPlanService { + public static final String INGESTOR_PROP_PREFIX = "ranger.audit.ingestor"; + + private final Properties configProps; + private final PartitionPlanHolder holder; + private final PartitionPlanRegistryFactory registryFactory; + private final KafkaAuditTopicPartitionGrower auditTopicPartitionGrower; + private final boolean dynamicPartitionPlanEnabled; + private final Set partitionPlanAdminUsers; + + public PartitionPlanService() { + this(AuditServerConfig.getInstance().getProperties(), PartitionPlanHolder.getInstance(), new PartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + } + + PartitionPlanService(Properties configProps, PartitionPlanHolder holder, PartitionPlanRegistryFactory registryFactory, KafkaAuditTopicPartitionGrower auditTopicPartitionGrower) { + this.configProps = configProps; + this.holder = holder; + this.registryFactory = registryFactory; + this.auditTopicPartitionGrower = auditTopicPartitionGrower; + this.dynamicPartitionPlanEnabled = PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configProps, INGESTOR_PROP_PREFIX); + this.partitionPlanAdminUsers = cachePartitionPlanAdminUsers(configProps); + } + + /** Returns whether dynamic partition-plan mode is enabled in ingestor configuration. */ + public boolean isDynamicPartitionPlanEnabled() { + return dynamicPartitionPlanEnabled; + } + + /** Returns the plan currently installed in memory on this ingestor pod. */ + public PartitionPlan getPartitionPlan() { + PartitionPlan plan = holder.getPlan(); + if (plan == null) { + throw new PartitionPlanException("Partition plan is not loaded in memory"); + } + return plan; + } + + /** Onboards a plugin: promote from buffer and register service allowlists atomically. */ + public PartitionPlan onboardPlugin(OnboardPlugin onboardPluginRequest, String updatedBy) { + PartitionPlanRequestValidator.validateOnboardPlugin(onboardPluginRequest); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + requireExpectedVersion(currentPlan, onboardPluginRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isOnboardAlreadyApplied(currentPlan, onboardPluginRequest.getPluginId(), onboardPluginRequest.getPartitionCount(), onboardPluginRequest.getServices())) { + return returnCurrentPlanNoOp(currentPlan); + } + PartitionPlan nextPlan = PartitionPlanAllocator.onboardPlugin(currentPlan, onboardPluginRequest.getPluginId(), onboardPluginRequest.getPartitionCount(), onboardPluginRequest.getServices(), updatedBy); + return publishMutation(registry, auditTopic, onboardPluginRequest.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to onboard plugin in partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Updates an onboarded plugin: scale and/or mutate service allowlists in one plan version. */ + public PartitionPlan updatePlugin(String pluginId, UpdatePlugin updatePluginRequest, String updatedBy) { + PartitionPlanRequestValidator.validateUpdatePlugin(pluginId, updatePluginRequest); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + requireExpectedVersion(currentPlan, updatePluginRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isUpdateAlreadyApplied(currentPlan, pluginId, updatePluginRequest)) { + return returnCurrentPlanNoOp(currentPlan); + } + PartitionPlan nextPlan = PartitionPlanAllocator.updatePlugin(currentPlan, pluginId, updatePluginRequest, updatedBy); + return publishMutation(registry, auditTopic, updatePluginRequest.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to update plugin in partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Returns configured admin short usernames for partition-plan REST (empty = not restricted beyond authentication). */ + public Set getPartitionPlanAdminUsers() { + return partitionPlanAdminUsers; + } + + private static Set cachePartitionPlanAdminUsers(Properties configProps) { + Set adminUsers = PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); + Set ret = adminUsers; + + if (!adminUsers.isEmpty()) { + ret = Collections.unmodifiableSet(adminUsers); + } + + return ret; + } + + /** Validates version, grows the audit topic if needed, writes the plan, and reloads memory. */ + private PartitionPlan publishMutation(PartitionPlanRegistry registry, String auditTopic, int expectedVersion, PartitionPlan current, PartitionPlan next) { + requireExpectedVersion(current, expectedVersion); + growAuditTopicIfNeeded(next.getTopicPartitionCount()); + verifyVersionUnchanged(registry, expectedVersion); + registry.writePlan(auditTopic, next); + verifyReadback(registry, auditTopic, next.getVersion()); + return holder.getPlan(); + } + + /** Returns the current plan without a registry write when the desired state is already satisfied. */ + private PartitionPlan returnCurrentPlanNoOp(PartitionPlan current) { + holder.install(current, current.getTopicPartitionCount()); + return holder.getPlan(); + } + + private static void requireExpectedVersion(PartitionPlan current, int expectedVersion) { + if (current.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(current); + } + } + + /** Grows the audit topic before the plan references new partition IDs. */ + private void growAuditTopicIfNeeded(int requiredPartitions) { + try { + auditTopicPartitionGrower.growAuditTopicToRequiredPartitionCount(configProps, INGESTOR_PROP_PREFIX, resolveAuditTopicName(), requiredPartitions); + } catch (RuntimeException e) { + throw new PartitionPlanException("Failed to grow audit topic partition count", e); + } + } + + /** Confirms the registry still holds the expected version before writing. */ + private void verifyVersionUnchanged(PartitionPlanRegistry registry, int expectedVersion) { + PartitionPlan latest = registry.readPlan(resolveAuditTopicName()); + if (latest == null) { + throw new PartitionPlanException("Partition plan disappeared during update"); + } + if (latest.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(latest); + } + } + + /** Mandatory read-back after publish so every pod converges on the same plan version. */ + private void verifyReadback(PartitionPlanRegistry registry, String auditTopic, int expectedVersion) { + PartitionPlan readback = registry.readPlan(auditTopic); + if (readback == null || readback.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(readback != null ? readback : holder.getPlan()); + } + holder.install(readback, readback.getTopicPartitionCount()); + } + + /** Loads the current plan from Kafka or fails when the registry is empty. */ + private static PartitionPlan requirePlan(PartitionPlanRegistry registry, String auditTopic) { + PartitionPlan plan = registry.readPlan(auditTopic); + if (plan == null) { + throw new PartitionPlanException("No partition plan found in Kafka for audit topic '" + auditTopic + "'"); + } + return plan; + } + + /** Rejects REST calls when dynamic mode is disabled. */ + private void requireDynamicEnabled() { + if (!isDynamicPartitionPlanEnabled()) { + throw new PartitionPlanException("Dynamic partition plan is not enabled"); + } + } + + /** Resolves the audit data topic name from ingestor configuration. */ + private String resolveAuditTopicName() { + return MiscUtil.getStringProperty(configProps, INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java new file mode 100644 index 0000000000..0ec280ecc8 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** Applies compacted partition-plan Kafka records into {@link PartitionPlanHolder}. */ +public class PartitionPlanUpdateApplier { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanUpdateApplier.class); + + @FunctionalInterface + public interface AuditTopicPartitionCountSupplier { + int getPartitionCount() throws Exception; + } + + private final Properties props; + private final String auditTopicKey; + private final PartitionPlanHolder holder; + private final AuditTopicPartitionCountSupplier partitionCountSupplier; + + public PartitionPlanUpdateApplier( + Properties props, + String auditTopicKey, + PartitionPlanHolder holder, + AuditTopicPartitionCountSupplier partitionCountSupplier) { + this.props = props; + this.auditTopicKey = auditTopicKey; + this.holder = holder != null ? holder : PartitionPlanHolder.getInstance(); + this.partitionCountSupplier = partitionCountSupplier; + } + + /** Installs a plan record when its version is newer than the in-memory copy. */ + public void applyRecordIfNewer(ConsumerRecord record) { + if (!auditTopicKey.equals(record.key())) { + return; + } + try { + PartitionPlan plan = PartitionPlan.fromJson(record.value()); + plan = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + if (plan.getVersion() <= holder.getLastInstalledVersion()) { + return; + } + holder.install(plan, partitionCountSupplier.getPartitionCount()); + LOG.info("Installed partition plan version {} from Kafka offset {}", plan.getVersion(), record.offset()); + } catch (Exception e) { + LOG.error("Ignoring invalid partition plan at offset {} for audit topic '{}'", record.offset(), auditTopicKey, e); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java new file mode 100644 index 0000000000..163c416105 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Checks partition plan shape and append-only updates. */ +public class PartitionPlanValidator { + private PartitionPlanValidator() { + } + + public static void validate(PartitionPlan plan) { + validate(plan, null); + } + + /** When kafkaPartitionCount is set, it must match plan.topicPartitionCount. */ + public static void validate(PartitionPlan plan, Integer kafkaPartitionCount) { + if (plan == null || StringUtils.isBlank(plan.getTopic()) || plan.getVersion() < PartitionPlanConstants.INITIAL_PLAN_VERSION || plan.getTopicPartitionCount() < 1) { + throw new PartitionPlanException("Invalid partition plan"); + } + if (kafkaPartitionCount != null && !kafkaPartitionCount.equals(plan.getTopicPartitionCount())) { + throw new PartitionPlanException("topicPartitionCount does not match Kafka topic partition count"); + } + + Set assigned = new HashSet<>(); + registerPartitions(plan.getBuffer().getPartitions(), plan.getTopicPartitionCount(), assigned, true); + for (Map.Entry entry : plan.getPlugins().entrySet()) { + if (StringUtils.isBlank(entry.getKey())) { + throw new PartitionPlanException("Plugin id is required"); + } + registerPartitions(entry.getValue().getPartitions(), plan.getTopicPartitionCount(), assigned, false); + } + if (assigned.size() != plan.getTopicPartitionCount()) { + throw new PartitionPlanException("Partition plan must assign every topic partition exactly once"); + } + validateServices(plan.getServices()); + } + + /** When present, each service entry must declare at least one allowed short username. */ + public static void validateServices(Map services) { + if (services == null || services.isEmpty()) { + return; + } + for (Map.Entry entry : services.entrySet()) { + if (StringUtils.isBlank(entry.getKey())) { + throw new PartitionPlanException("Service repo name is required"); + } + ServiceAllowlistEntry allowlistEntry = entry.getValue(); + if (allowlistEntry == null || allowlistEntry.getAllowedUsers().isEmpty()) { + throw new PartitionPlanException("allowedUsers must not be empty for service '" + entry.getKey() + "'"); + } + } + } + + /** New plan must only add tail partitions; existing plugin lists stay unchanged in order. */ + public static void validateAppendOnly(PartitionPlan current, PartitionPlan proposed) { + if (current == null || proposed == null) { + throw new PartitionPlanException("Current and proposed plans are required"); + } + if (proposed.getTopicPartitionCount() < current.getTopicPartitionCount() || proposed.getVersion() != current.getVersion() + 1) { + throw new PartitionPlanException("Plan must grow partition count and increment version by one"); + } + + for (Map.Entry entry : current.getPlugins().entrySet()) { + String pluginId = entry.getKey(); + List before = entry.getValue().getPartitions(); + PluginPartitionAssignment afterAssignment = proposed.getPlugins().get(pluginId); + if (afterAssignment == null) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "'"); + } + List after = afterAssignment.getPartitions(); + if (after.size() < before.size()) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "'"); + } + for (int i = 0; i < before.size(); i++) { + if (!before.get(i).equals(after.get(i))) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "' at index " + i); + } + } + } + } + + private static void registerPartitions(List partitionIds, int topicPartitionCount, Set assigned, boolean allowEmpty) { + if (partitionIds.isEmpty()) { + if (allowEmpty) { + return; + } + throw new PartitionPlanException("Plugin partition list must not be empty"); + } + for (int partitionId : partitionIds) { + if (partitionId < 0 || partitionId >= topicPartitionCount || !assigned.add(partitionId)) { + throw new PartitionPlanException("Invalid or duplicate partition id: " + partitionId); + } + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java new file mode 100644 index 0000000000..18ed914447 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +/** Background thread: load plan from Kafka (or XML-seeded initial bootstrap plan), then incrementally refresh in-memory plan. */ +public class PartitionPlanWatcher implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanWatcher.class); + + private final Properties props; + private final String propPrefix; + private final String auditTopicKey; + private final PartitionPlanHolder partitionPlanHolder; + private final int refreshIntervalMs; + private final int consumerPollTimeoutMs; + private final String planTopic; + + private PartitionPlanRegistry registry; + private KafkaConsumer consumer; + private PartitionPlanUpdateApplier planUpdateApplier; + private Thread watcherThread; + private volatile boolean running; + + public PartitionPlanWatcher(Properties props, String propPrefix, String auditTopicKey, PartitionPlanHolder holder) { + this.props = props; + this.propPrefix = propPrefix; + this.auditTopicKey = auditTopicKey; + this.partitionPlanHolder = holder != null ? holder : PartitionPlanHolder.getInstance(); + this.refreshIntervalMs = PartitionPlanKafkaConfig.resolveRefreshIntervalMs(props, propPrefix); + this.consumerPollTimeoutMs = PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, propPrefix); + this.planTopic = PartitionPlanKafkaConfig.resolvePlanTopicName(props, propPrefix); + this.planUpdateApplier = new PartitionPlanUpdateApplier(props, auditTopicKey, this.partitionPlanHolder, this::resolveAuditTopicPartitionCount); + } + + /** Blocking startup: empty-registry bootstrap, install plan, then start background refresh. */ + public void startBlocking() throws Exception { + LOG.info("Starting partition plan watcher for audit topic '{}' (plan topic '{}', refresh {} ms, consumer poll {} ms)", auditTopicKey, planTopic, refreshIntervalMs, consumerPollTimeoutMs); + registry = new KafkaPartitionPlanRegistry(props, propPrefix); + Map producerConfig = buildProducerConfigMap(); + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, auditTopicKey, producerConfig); + plan = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + int kafkaPartitionCount = resolveAuditTopicPartitionCount(); + partitionPlanHolder.install(plan, kafkaPartitionCount); + openConsumerAtBeginning(); + pollIncrementalUpdates(); + running = true; + watcherThread = new Thread(this, "PartitionPlanWatcher"); + watcherThread.setDaemon(true); + watcherThread.start(); + LOG.info("Partition plan watcher ready: version={}, auditTopicPartitions={}", plan.getVersion(), kafkaPartitionCount); + } + + /** Stops the background refresh thread and closes Kafka clients. */ + public void stop() { + running = false; + if (watcherThread != null) { + watcherThread.interrupt(); + try { + watcherThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + watcherThread = null; + } + closeConsumer(); + if (registry != null) { + registry.close(); + registry = null; + } + LOG.info("Partition plan watcher stopped"); + } + + @Override + public void run() { + while (running) { + try { + pollIncrementalUpdates(); + Thread.sleep(refreshIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LOG.error("Partition plan refresh failed; keeping last known good plan version {}", partitionPlanHolder.getLastInstalledVersion(), e); + } + } + } + + /** Drains new compacted plan records and installs any newer version into memory. */ + private void pollIncrementalUpdates() { + if (consumer == null) { + return; + } + ConsumerRecords records; + do { + records = consumer.poll(Duration.ofMillis(consumerPollTimeoutMs)); + for (ConsumerRecord record : records) { + planUpdateApplier.applyRecordIfNewer(record); + } + } while (!records.isEmpty()); + } + + private void openConsumerAtBeginning() throws Exception { + consumer = new KafkaConsumer<>(PartitionPlanKafkaConfig.consumerConfig(props, propPrefix, PartitionPlanConstants.PLAN_WATCHER_CONSUMER_GROUP)); + TopicPartition partition = new TopicPartition(planTopic, 0); + consumer.assign(Collections.singletonList(partition)); + consumer.seekToBeginning(Collections.singletonList(partition)); + } + + private void closeConsumer() { + if (consumer != null) { + consumer.close(); + consumer = null; + } + } + + private int resolveAuditTopicPartitionCount() throws Exception { + try { + Map adminConfig = AuditMessageQueueUtils.buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singletonList(auditTopicKey)); + TopicDescription topicDescription = describeTopicsResult.values().get(auditTopicKey).get(); + return topicDescription.partitions().size(); + } + } catch (Exception e) { + LOG.error("Failed to resolve partition count for audit topic '{}'", auditTopicKey, e); + throw e; + } + } + + private Map buildProducerConfigMap() { + Map producerConfig = new LinkedHashMap<>(); + String prefix = propPrefix + "."; + for (String key : props.stringPropertyNames()) { + if (key.startsWith(prefix)) { + producerConfig.put(key, props.getProperty(key)); + } + } + return producerConfig; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java new file mode 100644 index 0000000000..43e6b32efa --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +/** One primary {@code auth_to_local} catalog rule and its mapped Kerberos short name. */ +public class PrimaryCatalogRule { + final String ruleLine; + final String mappedShortName; + + PrimaryCatalogRule(String ruleLine, String mappedShortName) { + this.ruleLine = ruleLine; + this.mappedShortName = mappedShortName; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java new file mode 100644 index 0000000000..1ef0ef7731 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_SUFFIX_ALLOWED_USERS; + +/** Loads service allowlist entries from ingestor site XML for registry bootstrap and brownfield merge. */ +public class ServiceAllowlistBootstrap { + private static final String ALLOWLIST_SOURCE_SITE_XML = "xml-bootstrap"; + + private ServiceAllowlistBootstrap() { + } + + /** Scans {@code ranger.audit.ingestor.service..allowed.users} properties. */ + public static Map loadAllowlistsFromProperties(Properties ingestorProperties) { + Map allowlistEntriesByRepo = new LinkedHashMap<>(); + if (ingestorProperties == null) { + return allowlistEntriesByRepo; + } + for (String propertyName : ingestorProperties.stringPropertyNames()) { + if (!propertyName.startsWith(PROP_PREFIX_AUDIT_SERVER_SERVICE) || !propertyName.endsWith(PROP_SUFFIX_ALLOWED_USERS)) { + continue; + } + String serviceRepoName = propertyName.substring(PROP_PREFIX_AUDIT_SERVER_SERVICE.length(), propertyName.length() - PROP_SUFFIX_ALLOWED_USERS.length()); + if (StringUtils.isBlank(serviceRepoName)) { + continue; + } + String allowedUsersPropertyValue = ingestorProperties.getProperty(propertyName); + if (StringUtils.isBlank(allowedUsersPropertyValue)) { + continue; + } + List allowedUserShortNames = parseAllowedUserShortNames(allowedUsersPropertyValue); + if (allowedUserShortNames.isEmpty()) { + continue; + } + allowlistEntriesByRepo.put( + serviceRepoName.trim(), + new ServiceAllowlistEntry(allowedUserShortNames, ALLOWLIST_SOURCE_SITE_XML, null, null)); + } + return allowlistEntriesByRepo; + } + + /** Loads allowlist entries from the running ingestor configuration singleton. */ + public static Map loadAllowlistsFromServerConfig() { + return loadAllowlistsFromProperties(AuditServerConfig.getInstance().getProperties()); + } + + /** + * Brownfield helper: when the Kafka plan has no {@code services} block, merge XML entries in memory only + * (does not bump version or write to Kafka). + */ + public static PartitionPlan mergeSiteXmlAllowlistsWhenPlanServicesMissing( + PartitionPlan partitionPlan, Properties ingestorProperties) { + PartitionPlan ret = partitionPlan; + + if (partitionPlan != null && (partitionPlan.getServices() == null || partitionPlan.getServices().isEmpty())) { + Map siteXmlAllowlistEntries = loadAllowlistsFromProperties(ingestorProperties); + + if (!siteXmlAllowlistEntries.isEmpty()) { + ret = partitionPlan.toBuilder().services(siteXmlAllowlistEntries).build(); + } + } + + return ret; + } + + private static List parseAllowedUserShortNames(String allowedUsersPropertyValue) { + List allowedUserShortNames = new ArrayList<>(); + for (String userToken : allowedUsersPropertyValue.split(",")) { + if (userToken != null) { + String trimmedShortName = userToken.trim(); + if (StringUtils.isNotBlank(trimmedShortName)) { + allowedUserShortNames.add(trimmedShortName); + } + } + } + return allowedUserShortNames; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java new file mode 100644 index 0000000000..59080afc21 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.Set; + +/** + * Per-repo audit POST authorization after {@code auth_to_local} mapping. + * + *

Distinct from the global allowlist union in {@link AuthToLocalRuleComposer#collectAllowedUserShortNames} + * (which selects Kerberos mapping rules). Checks whether the mapped short name is allowed for the + * specific {@code serviceName} on {@code POST /api/audit/access?serviceName=...}. + * + *

Example (dynamic mode, plan has dev_hdfs with hdfs and nn only): + *

    + *
  • isAllowed(dev_hdfs, hdfs) returns true (repo in plan)
    • + *
  • isAllowed(dev_hive, hive) returns false unless static XML lists hive for dev_hive + * (repo not in plan -> registry returns null -> XML fallback)
    • + *
+ */ +public class ServiceAllowlistResolver { + private ServiceAllowlistResolver() { + } + + /** + * @param serviceName Ranger repo from {@code serviceName} query param (e.g. {@code dev_hdfs}) + * @param userName short name after {@code KerberosName.getShortName()} (e.g. {@code hdfs} from {@code nn/...}) + * @param holder partition-plan registry; per-repo {@code services[serviceName].allowedUsers} + */ + public static boolean isAllowedServiceUser(String serviceName, String userName, boolean dynamicPartitionPlanEnabled, PartitionPlanHolder holder, Map> staticAllowedUsersByService) { + boolean ret = false; + + if (!StringUtils.isBlank(serviceName) && !StringUtils.isBlank(userName)) { + if (dynamicPartitionPlanEnabled && holder != null) { + Set registryUsers = holder.getAllowedUsersForService(serviceName); + + if (registryUsers != null) { + ret = registryUsers.contains(userName); + } else { + Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; + ret = allowedUsers != null && allowedUsers.contains(userName); + } + } else { + Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; + ret = allowedUsers != null && allowedUsers.contains(userName); + } + } + + return ret; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java new file mode 100644 index 0000000000..13436b3448 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.constants; + +/** Shared constants for the dynamic Kafka partition-plan registry. */ +public class PartitionPlanConstants { + /** Version number of the first XML-seeded initial bootstrap plan in an empty registry. */ + public static final int INITIAL_PLAN_VERSION = 1; + /** {@code updatedBy} value on plans published by {@code PartitionPlanBootstrap}. */ + public static final String BOOTSTRAP_UPDATED_BY = "bootstrap"; + public static final String PLAN_REGISTRY_CONSUMER_GROUP = "ranger_audit_partition_plan_registry"; + public static final String PLAN_WATCHER_CONSUMER_GROUP = "ranger_audit_partition_plan_watcher"; + + private PartitionPlanConstants() { + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java new file mode 100644 index 0000000000..7d22de9f7f --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.exception; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; + +/** Optimistic-lock failure: another writer changed the plan version first (HTTP 409). */ +public final class PartitionPlanConflictException extends PartitionPlanException { + private static final long serialVersionUID = 1L; + + private final PartitionPlan currentPlan; + + public PartitionPlanConflictException(PartitionPlan currentPlan) { + super("Partition plan version conflict"); + this.currentPlan = currentPlan; + } + + public PartitionPlan getCurrentPlan() { + return currentPlan; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java new file mode 100644 index 0000000000..9080e4b238 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.exception; + +/** + * Raised when a partition plan is invalid or an append-only allocation cannot be applied. + */ +public class PartitionPlanException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public PartitionPlanException(String message) { + super(message); + } + + public PartitionPlanException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java new file mode 100644 index 0000000000..1f21b7b57a --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Request body for POST /api/audit/partition-plan/plugins. {@code services} is required (non-empty) at validation. */ +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class OnboardPlugin implements Serializable { + private final String pluginId; + private final int partitionCount; + private final int expectedVersion; + private final Map services; + + @JsonCreator + public OnboardPlugin(@JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("services") Map services) { + this.pluginId = pluginId; + this.partitionCount = partitionCount; + this.expectedVersion = expectedVersion; + this.services = copyServices(services); + } + + public OnboardPlugin(String pluginId, int partitionCount, int expectedVersion) { + this(pluginId, partitionCount, expectedVersion, null); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public String getPluginId() { + return pluginId; + } + + public int getPartitionCount() { + return partitionCount; + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public Map getServices() { + return services; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java new file mode 100644 index 0000000000..40794e976c --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanValidator; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.provider.MiscUtil; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PartitionPlan implements java.io.Serializable { + private final String topic; + private final int version; + private final int topicPartitionCount; + private final String updatedAt; + private final String updatedBy; + private final Map plugins; + private final PluginPartitionAssignment buffer; + private final Map services; + + @JsonCreator + public PartitionPlan(@JsonProperty("topic") String topic, @JsonProperty("version") int version, @JsonProperty("topicPartitionCount") int topicPartitionCount, @JsonProperty("updatedAt") String updatedAt, @JsonProperty("updatedBy") String updatedBy, @JsonProperty("plugins") Map plugins, @JsonProperty("buffer") PluginPartitionAssignment buffer, @JsonProperty("services") Map services) { + this.topic = topic; + this.version = version; + this.topicPartitionCount = topicPartitionCount; + this.updatedAt = updatedAt; + this.updatedBy = updatedBy; + this.plugins = copyPlugins(plugins); + this.buffer = buffer != null ? buffer : PluginPartitionAssignment.empty(); + this.services = copyServices(services); + } + + private static Map copyPlugins(Map plugins) { + if (plugins == null || plugins.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(plugins)); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public String getTopic() { + return topic; + } + + public int getVersion() { + return version; + } + + public int getTopicPartitionCount() { + return topicPartitionCount; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUpdatedBy() { + return updatedBy; + } + + public Map getPlugins() { + return plugins; + } + + public PluginPartitionAssignment getBuffer() { + return buffer; + } + + public Map getServices() { + return services; + } + + /** Compares routing payload; ignores version, updatedAt, and updatedBy. */ + public boolean sameContentAs(PartitionPlan other) { + if (other == null) { + return false; + } + return topicPartitionCount == other.topicPartitionCount + && Objects.equals(topic, other.topic) + && Objects.equals(plugins, other.plugins) + && Objects.equals(buffer, other.buffer) + && Objects.equals(services, other.services); + } + + public Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + /** Serializes this plan for the compacted Kafka registry topic. */ + public String toJson() { + try { + return MiscUtil.getMapper().writeValueAsString(this); + } catch (Exception e) { + throw new PartitionPlanException("Failed to serialize partition plan", e); + } + } + + /** Parses and validates a plan JSON payload from Kafka or REST. */ + public static PartitionPlan fromJson(String json) { + try { + PartitionPlan plan = MiscUtil.getMapper().readValue(json, PartitionPlan.class); + PartitionPlanValidator.validate(plan); + return plan; + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to deserialize partition plan", e); + } + } + + @Override + public boolean equals(Object otherPartitionPlanObj) { + if (this == otherPartitionPlanObj) { + return true; + } + if (otherPartitionPlanObj == null || getClass() != otherPartitionPlanObj.getClass()) { + return false; + } + PartitionPlan otherPartitionPlan = (PartitionPlan) otherPartitionPlanObj; + return version == otherPartitionPlan.version + && topicPartitionCount == otherPartitionPlan.topicPartitionCount + && Objects.equals(topic, otherPartitionPlan.topic) + && Objects.equals(updatedAt, otherPartitionPlan.updatedAt) + && Objects.equals(updatedBy, otherPartitionPlan.updatedBy) + && Objects.equals(plugins, otherPartitionPlan.plugins) + && Objects.equals(buffer, otherPartitionPlan.buffer) + && Objects.equals(services, otherPartitionPlan.services); + } + + @Override + public int hashCode() { + return Objects.hash(topic, version, topicPartitionCount, updatedAt, updatedBy, plugins, buffer, services); + } + + @Override + public String toString() { + return "PartitionPlan{topic='" + topic + "', version=" + version + ", topicPartitionCount=" + topicPartitionCount + ", plugins=" + plugins.keySet() + ", bufferSize=" + buffer.size() + ", services=" + services.keySet() + '}'; + } + + public static final class Builder { + private String topic; + private int version = 1; + private int topicPartitionCount; + private String updatedAt; + private String updatedBy; + private Map plugins = new LinkedHashMap<>(); + private PluginPartitionAssignment buffer = PluginPartitionAssignment.empty(); + private Map services = new LinkedHashMap<>(); + + private Builder() { + } + + private Builder(PartitionPlan plan) { + this.topic = plan.topic; + this.version = plan.version; + this.topicPartitionCount = plan.topicPartitionCount; + this.updatedAt = plan.updatedAt; + this.updatedBy = plan.updatedBy; + this.plugins = new LinkedHashMap<>(plan.plugins); + this.buffer = plan.buffer; + this.services = new LinkedHashMap<>(plan.services); + } + + public Builder topic(String topic) { + this.topic = topic; + return this; + } + + public Builder version(int version) { + this.version = version; + return this; + } + + public Builder topicPartitionCount(int topicPartitionCount) { + this.topicPartitionCount = topicPartitionCount; + return this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return this; + } + + public Builder updatedBy(String updatedBy) { + this.updatedBy = updatedBy; + return this; + } + + public Builder plugins(Map plugins) { + this.plugins = plugins == null ? new LinkedHashMap<>() : new LinkedHashMap<>(plugins); + return this; + } + + public Builder putPlugin(String pluginId, PluginPartitionAssignment assignment) { + this.plugins.put(pluginId, assignment); + return this; + } + + public Builder buffer(PluginPartitionAssignment buffer) { + this.buffer = buffer != null ? buffer : PluginPartitionAssignment.empty(); + return this; + } + + public Builder services(Map services) { + this.services = services == null ? new LinkedHashMap<>() : new LinkedHashMap<>(services); + return this; + } + + public PartitionPlan build() { + return new PartitionPlan(topic, version, topicPartitionCount, updatedAt, updatedBy, plugins, buffer, services); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java new file mode 100644 index 0000000000..50157c9d7a --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PluginPartitionAssignment implements java.io.Serializable { + private final List partitions; + + @JsonCreator + public PluginPartitionAssignment(@JsonProperty("partitions") List partitions) { + if (partitions == null || partitions.isEmpty()) { + this.partitions = Collections.emptyList(); + } else { + this.partitions = List.copyOf(partitions); + } + } + + /** Returns an assignment with no partition IDs (empty buffer). */ + public static PluginPartitionAssignment empty() { + return new PluginPartitionAssignment(Collections.emptyList()); + } + + /** Builds an assignment from explicit partition IDs. */ + public static PluginPartitionAssignment of(int... partitionIds) { + List ids = new ArrayList<>(partitionIds.length); + for (int id : partitionIds) { + ids.add(id); + } + return new PluginPartitionAssignment(ids); + } + + /** Builds a contiguous inclusive partition ID range. */ + public static PluginPartitionAssignment ofRange(int startInclusive, int endInclusive) { + if (endInclusive < startInclusive) { + throw new IllegalArgumentException("Invalid partition range: " + startInclusive + "-" + endInclusive); + } + List ids = new ArrayList<>(endInclusive - startInclusive + 1); + for (int i = startInclusive; i <= endInclusive; i++) { + ids.add(i); + } + return new PluginPartitionAssignment(ids); + } + + public List getPartitions() { + return partitions; + } + + public int size() { + return partitions.size(); + } + + @Override + public boolean equals(Object otherPluginPartitionAssignmentObj) { + if (this == otherPluginPartitionAssignmentObj) { + return true; + } + if (otherPluginPartitionAssignmentObj == null || getClass() != otherPluginPartitionAssignmentObj.getClass()) { + return false; + } + PluginPartitionAssignment otherAssignment = (PluginPartitionAssignment) otherPluginPartitionAssignmentObj; + return Objects.equals(partitions, otherAssignment.partitions); + } + + @Override + public int hashCode() { + return Objects.hash(partitions); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java new file mode 100644 index 0000000000..8ad7f9663d --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ServiceAllowlistEntry implements Serializable { + private final List allowedUsers; + private final String source; + private final String notes; + private final String pluginId; + + @JsonCreator + public ServiceAllowlistEntry(@JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("source") String source, @JsonProperty("notes") String notes, @JsonProperty("pluginId") String pluginId) { + this.allowedUsers = copyAllowedUsers(allowedUsers); + this.source = source; + this.notes = notes; + this.pluginId = pluginId; + } + + public static ServiceAllowlistEntry ofUsers(String... users) { + return new ServiceAllowlistEntry(List.of(users), null, null, null); + } + + public static ServiceAllowlistEntry ofUsers(List users) { + return new ServiceAllowlistEntry(users, null, null, null); + } + + public static ServiceAllowlistEntry ofUsers(List users, String pluginId) { + return new ServiceAllowlistEntry(users, null, null, pluginId); + } + + private static List copyAllowedUsers(List allowedUsers) { + if (allowedUsers == null || allowedUsers.isEmpty()) { + return Collections.emptyList(); + } + Set unique = new LinkedHashSet<>(); + for (String user : allowedUsers) { + if (user != null && !user.isBlank()) { + unique.add(user.trim()); + } + } + return List.copyOf(unique); + } + + public List getAllowedUsers() { + return allowedUsers; + } + + public String getSource() { + return source; + } + + public String getNotes() { + return notes; + } + + public String getPluginId() { + return pluginId; + } + + /** True when normalized allowedUsers match (ignores source, notes, and pluginId). */ + public boolean hasSameAllowedUsers(List users) { + return Objects.equals(allowedUsers, ofUsers(users).getAllowedUsers()); + } + + @Override + public boolean equals(Object otherServiceAllowlistEntryObj) { + if (this == otherServiceAllowlistEntryObj) { + return true; + } + if (otherServiceAllowlistEntryObj == null || getClass() != otherServiceAllowlistEntryObj.getClass()) { + return false; + } + ServiceAllowlistEntry otherAllowlistEntry = (ServiceAllowlistEntry) otherServiceAllowlistEntryObj; + return Objects.equals(allowedUsers, otherAllowlistEntry.allowedUsers) + && Objects.equals(source, otherAllowlistEntry.source) + && Objects.equals(notes, otherAllowlistEntry.notes) + && Objects.equals(pluginId, otherAllowlistEntry.pluginId); + } + + @Override + public int hashCode() { + return Objects.hash(allowedUsers, source, notes, pluginId); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java new file mode 100644 index 0000000000..1eb3cf4ec9 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class UpdatePlugin implements Serializable { + private final int expectedVersion; + private final Integer additionalPartitions; + private final Map addServices; + private final Map updateServices; + private final List removeServices; + + @JsonCreator + public UpdatePlugin(@JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("additionalPartitions") Integer additionalPartitions, @JsonProperty("addServices") Map addServices, @JsonProperty("updateServices") Map updateServices, @JsonProperty("removeServices") List removeServices) { + this.expectedVersion = expectedVersion; + this.additionalPartitions = additionalPartitions; + this.addServices = copyServices(addServices); + this.updateServices = copyServices(updateServices); + this.removeServices = removeServices == null ? Collections.emptyList() : List.copyOf(removeServices); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public Integer getAdditionalPartitions() { + return additionalPartitions; + } + + public Map getAddServices() { + return addServices; + } + + public Map getUpdateServices() { + return updateServices; + } + + public List getRemoveServices() { + return removeServices; + } + + /** True when at least one mutation field is present. */ + public boolean hasMutationDelta() { + boolean ret = false; + + if (additionalPartitions != null && additionalPartitions >= 1) { + ret = true; + } else if (!addServices.isEmpty()) { + ret = true; + } else if (!updateServices.isEmpty()) { + ret = true; + } else if (!removeServices.isEmpty()) { + ret = true; + } + + return ret; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 2ae7d14e0f..a55b5a34cf 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -23,6 +23,16 @@ import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.AuditDestinationMgr; +import org.apache.ranger.audit.producer.kafka.partition.AuthToLocalRuleComposer; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanService; +import org.apache.ranger.audit.producer.kafka.partition.ServiceAllowlistResolver; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; @@ -35,8 +45,10 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.PATCH; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; @@ -68,6 +80,9 @@ public class AuditREST { @Autowired AuditDestinationMgr auditDestinationMgr; + @Autowired + PartitionPlanService partitionPlanService; + /** * Health check endpoint */ @@ -147,7 +162,7 @@ public Response getStatus() { } /** - * Access Audits producer endpoint. + * Access Audits producer endpoint. * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) * @param appId Optional query parameter for batch processing - identifies the application instance * @param accessAudits List of audit events to process @@ -240,6 +255,172 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q return ret; } + /** Returns the in-memory partition plan when dynamic mode is enabled. */ + @GET + @Path("/partition-plan") + @Produces("application/json") + public Response getPartitionPlan(@Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.getPartitionPlan()"); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("GET /partition-plan"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "GET /partition-plan"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = Response.ok(partitionPlanService.getPartitionPlan().toJson()).build(); + } catch (PartitionPlanException e) { + LOG.error("Partition plan GET failed", e); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(buildErrorResponse(e.getMessage())).build(); + } + } + } + LOG.debug("<== AuditREST.getPartitionPlan(): status={}", ret.getStatus()); + return ret; + } + + /** Onboards a plugin from the buffer and registers service allowlists. */ + @POST + @Path("/partition-plan/plugins") + @Consumes("application/json") + @Produces("application/json") + public Response onboardPlugin(OnboardPlugin request, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.onboardPlugin(pluginId={})", request != null ? request.getPluginId() : null); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("POST /partition-plan/plugins"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "POST /partition-plan/plugins"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.onboardPlugin(request, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("POST /partition-plan/plugins", e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("POST /partition-plan/plugins", e); + } catch (Exception e) { + LOG.error("Unexpected error onboarding plugin in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to onboard plugin in partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.onboardPlugin(): status={}", ret.getStatus()); + return ret; + } + + /** Updates an onboarded plugin: scale partitions and/or mutate service allowlists. */ + @PATCH + @Path("/partition-plan/plugins/{pluginId}") + @Consumes("application/json") + @Produces("application/json") + public Response updatePlugin(@PathParam("pluginId") String pluginId, UpdatePlugin updateRequest, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.updatePlugin(pluginId={})", pluginId); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("PATCH /partition-plan/plugins/{pluginId}"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "PATCH /partition-plan/plugins/{pluginId}"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.updatePlugin(pluginId, updateRequest, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("PATCH /partition-plan/plugins/" + pluginId, e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("PATCH /partition-plan/plugins/" + pluginId, e); + } catch (Exception e) { + LOG.error("Unexpected error updating plugin in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to update plugin in partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.updatePlugin(): status={}", ret.getStatus()); + return ret; + } + + /** Returns HTTP 200 with the updated plan JSON body. */ + private Response toSuccessfulPartitionPlanResponse(PartitionPlan updatedPlan) { + return Response.ok(updatedPlan.toJson()).build(); + } + + /** Returns HTTP 409 with the current plan when optimistic locking fails. */ + private Response toPartitionPlanConflictResponse(String operation, PartitionPlanConflictException conflict) { + PartitionPlan currentPlan = conflict.getCurrentPlan(); + if (currentPlan == null) { + LOG.error("{} rejected: partition plan version conflict (current plan unavailable)", operation, conflict); + return Response.status(Response.Status.CONFLICT).entity(buildErrorResponse("Partition plan version conflict")).build(); + } + LOG.error("{} rejected: partition plan version conflict; current version is {}", operation, currentPlan.getVersion(), conflict); + return Response.status(Response.Status.CONFLICT).entity(currentPlan.toJson()).build(); + } + + /** Returns HTTP 400 or 503 for validation and Kafka admin failures. */ + private Response toPartitionPlanErrorResponse(String operation, PartitionPlanException error) { + Response.Status status = resolvePartitionPlanErrorStatus(error); + LOG.error("{} failed: {}", operation, error.getMessage(), error); + return Response.status(status).entity(buildErrorResponse(error.getMessage())).build(); + } + + /** Returns HTTP 503 when dynamic partition-plan mode is disabled. */ + private Response partitionPlanDisabled(String operation) { + LOG.error("{} rejected: dynamic partition plan is not enabled", operation); + return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(buildErrorResponse("Dynamic partition plan is not enabled")).build(); + } + + /** + * When {@code kafka.partition.plan.allowed.users} is configured, restrict partition-plan REST to those short names. + * When unset, any authenticated principal may call partition-plan (backward compatible). + */ + private Response authorizePartitionPlanAdmin(HttpServletRequest request, String operation) { + Response ret = null; + String user = getAuthenticatedUser(request); + + if (StringUtils.isBlank(user)) { + LOG.error("{} rejected: authentication required", operation); + ret = Response.status(Response.Status.UNAUTHORIZED).entity(buildErrorResponse("Authentication required")).build(); + } else { + Set adminUsers = partitionPlanService.getPartitionPlanAdminUsers(); + + if (!adminUsers.isEmpty() && !adminUsers.contains(user)) { + LOG.error("{} rejected: user '{}' is not in partition plan admin allowlist", operation, user); + ret = Response.status(Response.Status.FORBIDDEN).entity(buildErrorResponse("User is not authorized to manage partition plan")).build(); + } + } + + return ret; + } + + /** Maps service/infrastructure failures to 503; client validation mistakes to 400. */ + private static Response.Status resolvePartitionPlanErrorStatus(PartitionPlanException error) { + Response.Status ret = Response.Status.BAD_REQUEST; + + if (error.getCause() != null) { + ret = Response.Status.SERVICE_UNAVAILABLE; + } else { + String message = error.getMessage(); + + if (message != null && (message.contains("Partition plan is not loaded in memory") + || message.contains("Partition plan disappeared during update") + || message.contains("No partition plan found in Kafka") + || message.contains("Mandatory read-back failed"))) { + ret = Response.Status.SERVICE_UNAVAILABLE; + } + } + + return ret; + } + + /** Records the authenticated admin user on plan mutations. */ + private String resolveUpdatedBy(HttpServletRequest request) { + String user = getAuthenticatedUser(request); + return StringUtils.isNotBlank(user) ? user : "rest-api"; + } + private String buildResponse(Map respMap) { try { return MiscUtil.getMapper().writeValueAsString(respMap); @@ -281,42 +462,48 @@ private String getAuthenticatedUser(HttpServletRequest request) { * For JWT or basic auth, the username is already in short form and returned as-is. */ private String applyAuthToLocal(String principal) { - if (StringUtils.isEmpty(principal)) { - return principal; - } + String ret = principal; - // Check if this looks like a Kerberos principal (has @ or /) - if (!principal.contains("@") && !principal.contains("/")) { - LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); - return principal; - } - - try { - KerberosName kerberosName = new KerberosName(principal); + if (StringUtils.isNotEmpty(principal)) { + // Check if this looks like a Kerberos principal (has @ or /) + if (!principal.contains("@") && !principal.contains("/")) { + LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); + } else { + try { + KerberosName kerberosName = new KerberosName(principal); - return kerberosName.getShortName(); - } catch (Exception e) { - LOG.warn("Failed to apply auth_to_local rules to principal '{}': {}. Using original principal.", principal, e.getMessage()); - return principal; + ret = kerberosName.getShortName(); + } catch (Exception e) { + LOG.warn("Failed to apply auth_to_local rules to principal '{}': {}. Using original principal.", principal, e.getMessage()); + } + } } + + return ret; } /** - * Rules are loaded from ranger.audit.ingestor.auth.to.local property in ranger-audit-ingestor-site.xml. + * Loads the auth_to_local catalog from ranger-audit-ingestor-site.xml. When dynamic partition-plan + * mode is disabled, applies the full catalog immediately. When enabled, applies static XML rules + * only if the partition-plan Kafka topic does not exist yet; otherwise composed rules are installed + * on {@link PartitionPlanHolder#install(PartitionPlan, Integer)} via {@link AuthToLocalRuleComposer#applyForPlan}. */ private static void initializeAuthToLocal() { AuditServerConfig config = AuditServerConfig.getInstance(); String authToLocalRules = config.get(AuditServerConstants.PROP_AUTH_TO_LOCAL); - if (StringUtils.isNotEmpty(authToLocalRules)) { - try { - KerberosName.setRules(authToLocalRules); - LOG.debug("Auth_to_local rules: {}", authToLocalRules); - } catch (Exception e) { - LOG.error("Failed to set auth_to_local rules from configuration: {}", e.getMessage(), e); - } - } else { + if (StringUtils.isEmpty(authToLocalRules)) { LOG.warn("No auth_to_local rules configured. Kerberos principal mapping may not work correctly."); LOG.warn("Set property '{}' in ranger-audit-ingestor-site.xml", AuditServerConstants.PROP_AUTH_TO_LOCAL); + return; + } + + AuthToLocalRuleComposer composer = AuthToLocalRuleComposer.getInstance(); + composer.initializeFromConfig(); + if (PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX)) { + composer.applyStartupRulesForDynamicMode(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX); + } else { + composer.applyStaticRules(); + LOG.debug("Applied static auth_to_local catalog from site XML"); } } @@ -327,18 +514,17 @@ private static void initializeAuthToLocal() { * @return true if user is allowed, false otherwise */ private boolean isAllowedServiceUser(String serviceName, String userName) { - boolean ret; - - if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) { - Set allowedUsers = allowedServiceUsers.get(serviceName); - - ret = allowedUsers != null && allowedUsers.contains(userName); - } else { - ret = false; - } - - LOG.debug("isAllowedServiceUser(serviceName={}, userName={}): ret={}", serviceName, userName, ret); - + boolean dynamicEnabled = partitionPlanService != null + && partitionPlanService.isDynamicPartitionPlanEnabled(); + boolean ret = ServiceAllowlistResolver.isAllowedServiceUser( + serviceName, + userName, + dynamicEnabled, + PartitionPlanHolder.getInstance(), + allowedServiceUsers); + LOG.debug( + "isAllowedServiceUser(serviceName={}, userName={}, dynamic={}): ret={}", + serviceName, userName, dynamicEnabled, ret); return ret; } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java index 185ca10000..ba25d93419 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java @@ -29,7 +29,8 @@ public class AuditServerConfig extends AuditConfig { private static final Logger LOG = LoggerFactory.getLogger(AuditServerConfig.class); - private static final String CONFIG_FILE_PATH = "conf/ranger-audit-ingestor-site.xml"; + private static final String CONFIG_FILE_PATH = "conf/ranger-audit-ingestor-site.xml"; + private static final String AUDIT_CONFIG_PROPERTY = "audit.config"; private static volatile AuditServerConfig sInstance; @@ -60,9 +61,10 @@ private boolean addAuditServerResources() { boolean ret = true; - // Load ranger-audit-ingestor-site.xml - if (!addAuditResource(CONFIG_FILE_PATH, true)) { - LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH); + // Prefer -Daudit.config (external conf dir) over classpath WEB-INF copy + String configPath = System.getProperty(AUDIT_CONFIG_PROPERTY, CONFIG_FILE_PATH); + if (!addAuditResource(configPath, true)) { + LOG.error("Could not load required configuration: {}", configPath); ret = false; } diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 1688b4dc43..52bb6594df 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -249,6 +249,21 @@ RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ + RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ + RULE:[2:$1/$2@$0](hbase/.*@.*)s/.*/hbase/ + RULE:[2:$1/$2@$0](kafka/.*@.*)s/.*/kafka/ + RULE:[2:$1/$2@$0](knox/.*@.*)s/.*/knox/ + RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ + RULE:[2:$1/$2@$0](trino/.*@.*)s/.*/trino/ + RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ + RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ + RULE:[2:$1/$2@$0](scm/.*@.*)s/.*/scm/ + RULE:[2:$1/$2@$0](dn/.*@.*)s/.*/dn/ + RULE:[2:$1/$2@$0](solr/.*@.*)s/.*/solr/ + RULE:[2:$1/$2@$0](rangertagsync/.*@.*)s/.*/rangertagsync/ + RULE:[2:$1/$2@$0](atlas/.*@.*)s/.*/atlas/ + RULE:[2:$1/$2@$0](kudu/.*@.*)s/.*/kudu/ + RULE:[2:$1/$2@$0](nifi/.*@.*)s/.*/nifi/ RULE:[1:$1@$0](.*@.*)s/@.*// DEFAULT @@ -265,6 +280,21 @@ - RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ # nn,dn,jn,hdfs/* -> hdfs (dev_hdfs) - RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ # rm,nm,yarn/* -> yarn (dev_yarn) - RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ # jhs/* -> mapred + - RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ # hive/* -> hive (dev_hive) + - RULE:[2:$1/$2@$0](hbase/.*@.*)s/.*/hbase/ # hbase/* -> hbase (dev_hbase) + - RULE:[2:$1/$2@$0](kafka/.*@.*)s/.*/kafka/ # kafka/* -> kafka (dev_kafka) + - RULE:[2:$1/$2@$0](knox/.*@.*)s/.*/knox/ # knox/* -> knox (dev_knox) + - RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ # rangerkms/* -> rangerkms (dev_kms) + - RULE:[2:$1/$2@$0](trino/.*@.*)s/.*/trino/ # trino/* -> trino (dev_trino) + - RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ # ozone/* -> ozone (dev_ozone) + - RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ # om/* -> om (dev_ozone) + - RULE:[2:$1/$2@$0](scm/.*@.*)s/.*/scm/ # scm/* -> scm (dev_ozone) + - RULE:[2:$1/$2@$0](dn/.*@.*)s/.*/dn/ # dn/* -> dn (dev_ozone) + - RULE:[2:$1/$2@$0](solr/.*@.*)s/.*/solr/ # solr/* -> solr (dev_solr) + - RULE:[2:$1/$2@$0](rangertagsync/.*@.*)s/.*/rangertagsync/ # rangertagsync/* -> rangertagsync (dev_tag) + - RULE:[2:$1/$2@$0](atlas/.*@.*)s/.*/atlas/ # atlas/* -> atlas (dev_atlas) + - RULE:[2:$1/$2@$0](kudu/.*@.*)s/.*/kudu/ # kudu/* -> kudu (dev_kudu) + - RULE:[2:$1/$2@$0](nifi/.*@.*)s/.*/nifi/ # nifi/* -> nifi (dev_nifi) - RULE:[1:$1@$0](.*@.*)s/@.*// # user@REALM -> user - DEFAULT ]]> @@ -322,12 +352,26 @@ ranger.audit.ingestor.kafka.configured.plugins - hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi + hdfs,yarn,knox,hive,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi - Comma-separated list of configured plugin IDs. - If set: Uses AuditPartitioner with auto-calculated partitions (sum of plugin allocations + buffer). - If empty/not set: Uses Kafka default hash-based partitioner with topic.partitions value. - Each plugin receives dedicated partitions based on topic.partitions.per.configured.plugin (default: 3). + Comma-separated plugin IDs (agentId / Kafka record key) that receive dedicated partition ranges in static mode. + Leave empty by default; add only the plugins you deploy. + + Empty (recommended greenfield): + - Static mode: Kafka default hash partitioner using kafka.topic.partitions below. + - Dynamic mode (kafka.partition.plan.dynamic.enabled=true): first bootstrap plan uses all + kafka.topic.partitions as buffer; onboard plugins via POST /api/audit/partition-plan/plugins. + + Non-empty (static / bootstrap seed): + - Uses AuditPartitioner: total partitions = sum(per-plugin counts) + kafka.topic.partitions.buffer. + - Append new plugin IDs only; never reorder existing entries (partition ranges are contiguous). + - Dynamic mode: seeds the initial plan from this list when ranger_audit_partition_plan is empty; + runtime changes use partition-plan REST, not XML edits. + + Example plugin IDs (pick what you run): hdfs, yarn, knox, hive, hiveServer2, hiveMetastore, kafka, + hbaseRegional, hbaseMaster, solr, trino, ozone, kudu, nifi. + Per-plugin partition count: kafka.topic.partitions.per.configured.plugin (default 3), overridable via + kafka.plugin.partition.overrides.{pluginId}. @@ -349,8 +393,8 @@ Number of buffer partitions reserved for unconfigured plugins. Used ONLY when configured.plugins is set (for plugin-based partitioning). Total = (sum of plugin partitions) + (buffer partitions). - Example: 7 plugins × 3 + 9 buffer = 30 total. - Ignored when configured.plugins is empty. + Example: 14 plugins × 3 + 9 buffer = 51 total. + Ignored when configured.plugins is empty (hash-based or dynamic buffer-only bootstrap). @@ -401,6 +445,45 @@ + + + ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled + false + + false or absent = legacy XML AuditPartitioner at startup (today's behavior). + true = Kafka registry + REST + PartitionPlanWatcher. + + + + ranger.audit.ingestor.kafka.security.protocol SASL_PLAINTEXT diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java new file mode 100644 index 0000000000..a45005c9c4 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrap; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrapConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AuditPartitionerDynamicTest { + private static final String TOPIC = "ranger_audits"; + private static final Node BROKER = new Node(0, "localhost", 9092); + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testStaticModeUnchangedWhenDynamicDisabled() { + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(staticProducerConfig()); + + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(1, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(2, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertTrue(partitioner.partition(TOPIC, "unknownPlugin", null, null, null, cluster(15)) >= 6); + } + + @Test + public void testDynamicModeRoutesConfiguredPluginRoundRobin() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + PartitionPlanHolder.getInstance().install(plan, 15); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(1, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(2, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertIterableEquals(List.of(3, 4, 5), List.of( + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)), + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)), + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)))); + } + + @Test + public void testDynamicModeRoutesUnknownPluginToBuffer() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 3)); + PartitionPlanHolder.getInstance().install(plan, 6); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + int partition = partitioner.partition(TOPIC, "trino", null, null, null, cluster(6)); + assertTrue(partition >= 3 && partition <= 5); + } + + @Test + public void testDynamicModeUsesPlannedTailPartitionWhenClusterMetadataLagsAfterScale() { + PartitionPlan planV1 = PartitionPlanBootstrap.createInitialPlan( + PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 9)); + PartitionPlanHolder.getInstance().install(planV1, 12); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + // Simulate scale: plan now assigns tail partitions 12,13 but producer metadata still shows 12 partitions. + PartitionPlan planV2 = planV1.toBuilder() + .version(2) + .topicPartitionCount(14) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2, 12, 13))) + .buffer(PluginPartitionAssignment.ofRange(3, 11)) + .updatedBy("test") + .build(); + PartitionPlanHolder.getInstance().install(planV2, 14); + + // Round-robin index 3 -> planned partition 12; must not clamp to 11 when cluster still reports 12 partitions. + for (int i = 0; i < 3; i++) { + partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(12)); + } + assertEquals(12, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(12))); + } + + @Test + public void testConcurrentPartitionWhilePlanSwaps() throws Exception { + PartitionPlan planV1 = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 2, 2)); + PartitionPlanHolder.getInstance().install(planV1, 4); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + Cluster cluster = cluster(4); + + PartitionPlan planV2 = planV1.toBuilder().version(2).updatedBy("test").build(); + + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch startLatch = new CountDownLatch(1); + AtomicInteger errors = new AtomicInteger(); + Set seen = Collections.synchronizedSet(new HashSet<>()); + + for (int i = 0; i < 8; i++) { + executor.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < 200; j++) { + int p = partitioner.partition(TOPIC, "hdfs", null, null, null, cluster); + seen.add(p); + } + } catch (Exception e) { + errors.incrementAndGet(); + } + }); + } + + startLatch.countDown(); + PartitionPlanHolder.getInstance().install(planV2, 4); + executor.shutdown(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + assertEquals(0, errors.get()); + assertTrue(seen.stream().allMatch(p -> p >= 0 && p < 4)); + } + + private static Map staticProducerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map config = new HashMap<>(); + config.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS, 15); + config.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + return config; + } + + private static Map dynamicProducerConfig() { + Map config = staticProducerConfig(); + config.put(AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.substring(0, AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.length() - 1) + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return config; + } + + private static Cluster cluster(int partitionCount) { + PartitionInfo[] partitionInfos = new PartitionInfo[partitionCount]; + for (int i = 0; i < partitionCount; i++) { + partitionInfos[i] = new PartitionInfo(TOPIC, i, BROKER, new Node[] {BROKER}, new Node[] {BROKER}); + } + return new Cluster("cluster", Collections.singletonList(BROKER), List.of(partitionInfos), Collections.emptySet(), Collections.emptySet()); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java new file mode 100644 index 0000000000..91c2323416 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AuthToLocalRuleComposerTest { + private static final String SAMPLE_CATALOG = + "RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ " + + "RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ " + + "RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ " + + "RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ " + + "RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ " + + "RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ " + + "RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ " + + "RULE:[1:$1@$0](.*@.*)s/@.*// " + + "DEFAULT"; + + private static final String SENTINEL_NN_RULE = + "RULE:[2:$1/$2@$0](nn/.*@.*)s/.*/startup-not-applied/ DEFAULT"; + + private AuthToLocalRuleComposer composer; + + @BeforeEach + public void setUp() { + composer = AuthToLocalRuleComposer.getInstance(); + composer.resetForTests(); + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + composer.initializeFromProperties(props); + } + + @AfterEach + public void tearDown() { + composer.resetForTests(); + PartitionPlanHolder.getInstance().resetForTests(); + AuditServerConfig.getInstance().set( + PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, + "false"); + } + + @Test + public void testComposeKmsOnlyIncludesRangerkmsRuleAndTail() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("rangerkms")); + + assertTrue(rules.contains("(rangerkms/.*@.*)s/.*/rangerkms/")); + assertFalse(rules.contains("(hive/.*@.*)s/.*/hive/")); + assertTrue(rules.contains("RULE:[1:$1@$0](.*@.*)s/@.*//")); + assertTrue(rules.endsWith("DEFAULT")); + } + + @Test + public void testComposeHdfsUsesCatalogRegexNotSimpleRule() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("hdfs")); + + assertTrue(rules.contains("([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/")); + assertFalse(rules.contains("(hdfs/.*@.*)s/.*/hdfs/")); + } + + @Test + public void testComposeOzoneIncludesMultipleShortNames() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("ozone", "om", "scm", "dn")); + + assertTrue(rules.contains("(ozone/.*@.*)s/.*/ozone/")); + assertTrue(rules.contains("(om/.*@.*)s/.*/om/")); + } + + @Test + public void testComposeUnknownShortNameGeneratesSimpleRule() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("myapp")); + + assertTrue(rules.contains("(myapp/.*@.*)s/.*/myapp/")); + } + + @Test + public void testCollectAllowedUserShortNamesFromPlan() { + Map services = new LinkedHashMap<>(); + services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); + services.put("dev_ozone", ServiceAllowlistEntry.ofUsers("om", "ozone")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("kms", PluginPartitionAssignment.of(0, 1))) + .buffer(PluginPartitionAssignment.of(2, 3, 4, 5)) + .services(services) + .build(); + + Set activeShortNames = AuthToLocalRuleComposer.collectAllowedUserShortNames(plan); + + assertEquals(Set.of("rangerkms", "om", "ozone"), activeShortNames); + } + + @Test + public void testKerberosMappingForComposedKmsRules() throws Exception { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("rangerkms")); + KerberosName.setRules(rules); + + assertEquals("rangerkms", new KerberosName("rangerkms/ranger-kms.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testKerberosMappingForComposedHdfsRules() throws Exception { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("hdfs")); + KerberosName.setRules(rules); + + assertEquals("hdfs", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenTopicMissingAppliesStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPartitionPlanTopicExistsTestOverride(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("hdfs", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenTopicExistsDefersStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPartitionPlanTopicExistsTestOverride(true); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenDisabledIsNoOp() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "false"); + composer.setPartitionPlanTopicExistsTestOverride(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testInstallPlanAppliesComposedAuthToLocalRules() throws Exception { + AuditServerConfig config = AuditServerConfig.getInstance(); + config.set(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + config.set(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + composer.initializeFromConfig(); + + Map services = new LinkedHashMap<>(); + services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(3) + .topicPartitionCount(6) + .plugins(Map.of("kms", PluginPartitionAssignment.of(0, 1))) + .buffer(PluginPartitionAssignment.of(2, 3, 4, 5)) + .services(services) + .build(); + + PartitionPlanHolder.getInstance().install(plan, 6); + + assertEquals("rangerkms", new KerberosName("rangerkms/ranger-kms.rangernw@EXAMPLE.COM").getShortName()); + assertThrows(KerberosName.NoMatchingRule.class, + () -> new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testExtractMappedShortNameFromCatalogLines() { + assertEquals("hdfs", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/")); + assertEquals("rangerkms", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/")); + assertEquals("mapred", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/")); + } + + private static Properties dynamicModeProps() { + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java new file mode 100644 index 0000000000..220366ac40 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanAllocatorTest { + private PartitionPlan initialPlan; + + @BeforeEach + public void setUp() { + initialPlan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + @Test + public void testPromotePluginFromBuffer() { + PartitionPlan next = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + + assertEquals(2, next.getVersion()); + assertEquals(15, next.getTopicPartitionCount()); + assertIterableEquals(List.of(6, 7, 8), next.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of(9, 10, 11, 12, 13, 14), next.getBuffer().getPartitions()); + assertIterableEquals(List.of(0, 1, 2), next.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(3, 4, 5), next.getPlugins().get("hiveServer2").getPartitions()); + } + + @Test + public void testOnboardPluginWithMultipleServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + + PartitionPlan next = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); + + assertEquals(2, next.getVersion()); + assertIterableEquals(List.of(6, 7, 8), next.getPlugins().get("trino").getPartitions()); + assertEquals("trino", next.getServices().get("dev_hive").getPluginId()); + assertEquals("trino", next.getServices().get("dev_hive2").getPluginId()); + assertIterableEquals(List.of("hive"), next.getServices().get("dev_hive").getAllowedUsers()); + assertIterableEquals(List.of("hive2"), next.getServices().get("dev_hive2").getAllowedUsers()); + } + + @Test + public void testUpdatePluginAddsAndRemovesServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + PartitionPlan onboarded = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); + + Map addServices = Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")); + UpdatePlugin update = new UpdatePlugin(onboarded.getVersion(), null, addServices, null, List.of("dev_hive2")); + PartitionPlan updated = PartitionPlanAllocator.updatePlugin(onboarded, "trino", update, "ops"); + + assertEquals(3, updated.getVersion()); + assertTrue(updated.getServices().containsKey("dev_hive")); + assertTrue(updated.getServices().containsKey("dev_hive3")); + assertFalse(updated.getServices().containsKey("dev_hive2")); + assertEquals("trino", updated.getServices().get("dev_hive3").getPluginId()); + } + + @Test + public void testUpdatePluginScalesViaAdditionalPartitions() { + UpdatePlugin update = new UpdatePlugin(initialPlan.getVersion(), 3, null, null, null); + PartitionPlan scaled = PartitionPlanAllocator.updatePlugin(initialPlan, "hiveServer2", update, "ops"); + + assertEquals(2, scaled.getVersion()); + assertEquals(18, scaled.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17), scaled.getPlugins().get("hiveServer2").getPartitions()); + } + + @Test + public void testPromotePluginGrowsTopicWhenBufferInsufficient() { + PartitionPlan next = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 12, "ops"); + + assertEquals(18, next.getTopicPartitionCount()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), next.getPlugins().get("trino").getPartitions()); + assertEquals(0, next.getBuffer().size()); + } + + @Test + public void testScalePluginAppendsTailOnly() { + PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + PartitionPlan scaled = PartitionPlanAllocator.scalePlugin(promoted, "hiveServer2", 3, "ops"); + + assertEquals(3, scaled.getVersion()); + assertEquals(18, scaled.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17), scaled.getPlugins().get("hiveServer2").getPartitions()); + assertIterableEquals(List.of(0, 1, 2), scaled.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8), scaled.getPlugins().get("trino").getPartitions()); + } + + @Test + public void testPromoteAlreadyConfiguredPluginFails() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.promotePlugin(initialPlan, "hdfs", 1, "ops")); + assertTrue(error.getMessage().contains("requested 1")); + } + + @Test + public void testIsOnboardAlreadyAppliedWhenPluginServicesAndCountMatch() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + PartitionPlan onboarded = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); + + assertTrue(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "trino", 3, services)); + assertFalse(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "trino", 5, services)); + } + + @Test + public void testPromoteConflictWhenPartitionCountDiffers() { + PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.promotePlugin(promoted, "trino", 5, "ops")); + + assertTrue(error.getMessage().contains("requested 5")); + } + + @Test + public void testUpdatePluginRejectsRemoveForForeignService() { + PartitionPlan withTaggedService = initialPlan.toBuilder() + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers(List.of("hive"), "hdfs"))) + .build(); + + UpdatePlugin update = new UpdatePlugin(withTaggedService.getVersion(), null, null, null, List.of("dev_hive")); + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.updatePlugin(withTaggedService, "hiveServer2", update, "ops")); + + assertTrue(error.getMessage().contains("belongs to plugin")); + } + + @Test + public void testScaleUnknownPluginFails() { + assertThrows(PartitionPlanException.class, () -> PartitionPlanAllocator.scalePlugin(initialPlan, "trino", 2, "ops")); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java new file mode 100644 index 0000000000..3a3878e553 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +public class PartitionPlanBootstrapSupportTest { + private static final String TOPIC = "ranger_audits"; + + @Test + public void testBootstrapReturnsExistingPlan() { + PartitionPlan existing = samplePlan(); + InMemoryPartitionPlanRegistry registry = new InMemoryPartitionPlanRegistry(existing); + Map config = producerConfig(); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, config); + + assertEquals(existing, plan); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testBootstrapPublishesV1WhenEmpty() { + InMemoryPartitionPlanRegistry registry = new InMemoryPartitionPlanRegistry(null); + Map config = producerConfig(); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, config); + + assertEquals(1, plan.getVersion()); + assertEquals(15, plan.getTopicPartitionCount()); + assertIterableEquals(plan.getPlugins().get("hdfs").getPartitions(), registry.readPlan(TOPIC).getPlugins().get("hdfs").getPartitions()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testBootstrapAdoptsPeerPlanBeforePublish() { + PartitionPlan peerPlan = samplePlan().toBuilder().updatedBy("peer").build(); + PeerPublishesOnSecondReadRegistry registry = new PeerPublishesOnSecondReadRegistry(peerPlan); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, producerConfig()); + + assertEquals(peerPlan, plan); + assertEquals(0, registry.getWriteCount()); + } + + private static Map producerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map config = new HashMap<>(); + config.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + config.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + return config; + } + + private static PartitionPlan samplePlan() { + return PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + private static final class PeerPublishesOnSecondReadRegistry implements PartitionPlanRegistry { + private final PartitionPlan peerPlan; + private int readCount; + private int writeCount; + + private PeerPublishesOnSecondReadRegistry(PartitionPlan peerPlan) { + this.peerPlan = peerPlan; + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + readCount++; + if (readCount >= 2) { + return peerPlan; + } + return null; + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + writeCount++; + } + + @Override + public void close() { + } + + private int getWriteCount() { + return writeCount; + } + } + + private static class InMemoryPartitionPlanRegistry implements PartitionPlanRegistry { + private final AtomicReference planRef; + private int writeCount; + + private InMemoryPartitionPlanRegistry(PartitionPlan initialPlan) { + this.planRef = new AtomicReference<>(initialPlan); + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + return planRef.get(); + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + planRef.set(plan); + writeCount++; + } + + @Override + public void close() { + } + + private int getWriteCount() { + return writeCount; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java new file mode 100644 index 0000000000..d642bcebb0 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +public class PartitionPlanBootstrapTest { + private static final String TOPIC = "ranger_audits"; + + @Test + public void testCreateInitialPlanMatchesStaticPartitionerLayout() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + + assertEquals(TOPIC, plan.getTopic()); + assertEquals(1, plan.getVersion()); + assertEquals(15, plan.getTopicPartitionCount()); + assertEquals(PartitionPlanConstants.BOOTSTRAP_UPDATED_BY, plan.getUpdatedBy()); + assertIterableEquals(List.of(0, 1, 2), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(3, 4, 5), plan.getPlugins().get("hiveServer2").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanHonorsPluginOverrides() { + PartitionPlanBootstrapConfig config = PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "trino"}, 3, 4).withPluginOverride("hdfs", 5); + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(config); + + assertEquals(12, plan.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2, 3, 4), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(5, 6, 7), plan.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of(8, 9, 10, 11), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanEmptyPluginsUsesHashBasedTopicPartitions() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan( + new PartitionPlanBootstrapConfig(TOPIC, new String[0], 3, 9, 10, Collections.emptyMap())); + + assertEquals(10, plan.getTopicPartitionCount()); + assertEquals(0, plan.getPlugins().size()); + assertIterableEquals(List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanFromProducerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map configs = new HashMap<>(); + configs.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + configs.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + configs.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlanFromProducerConfig(configs, TOPIC); + + assertEquals(15, plan.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), plan.getBuffer().getPartitions()); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java new file mode 100644 index 0000000000..b0040b7463 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanHolderTest { + private static final int TOPIC_PARTITIONS = 6; + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testGetAllowedUsersReturnsNullWhenNoPlanInstalled() { + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + } + + @Test + public void testGetAllowedUsersReturnsNullWhenServicesBlockMissing() { + PartitionPlan plan = basePlanBuilder() + .build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + } + + @Test + public void testGetAllowedUsersReturnsNullForRepoNotInPartialPlan() { + Map services = new LinkedHashMap<>(); + services.put("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs", "nn")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + assertIterableEquals(List.of("hdfs", "nn"), PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_trino")); + } + + @Test + public void testGetAllowedUsersReturnsRegistryUsersForOnboardedRepo() { + Map services = Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + Set allowed = PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive"); + + assertIterableEquals(List.of("hive"), allowed); + assertTrue(allowed != null && !allowed.contains("hdfs")); + } + + @Test + public void testInstallRejectsEmptyServiceAllowlist() { + Map services = Map.of("dev_hdfs", new ServiceAllowlistEntry(List.of(), "test", null, null)); + PartitionPlan plan = basePlanBuilder().services(services).build(); + + assertThrows(PartitionPlanException.class, () -> PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS)); + } + + private static PartitionPlan.Builder basePlanBuilder() { + return PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(TOPIC_PARTITIONS) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java new file mode 100644 index 0000000000..5b813f0dc3 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanKafkaConfigTest { + private static final String PROP_PREFIX = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER + "kafka"; + + @Test + public void testResolvePlanTopicNameUsesDefault() { + Properties props = new Properties(); + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC, PartitionPlanKafkaConfig.resolvePlanTopicName(props, PROP_PREFIX)); + } + + @Test + public void testResolvePlanTopicNameUsesOverride() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, "custom_plan_topic"); + assertEquals("custom_plan_topic", PartitionPlanKafkaConfig.resolvePlanTopicName(props, PROP_PREFIX)); + } + + @Test + public void testDynamicPartitionPlanDisabledByDefault() { + assertFalse(PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(new Properties(), PROP_PREFIX)); + } + + @Test + public void testDynamicPartitionPlanEnabledFromProperty() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + assertTrue(PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(props, PROP_PREFIX)); + } + + @Test + public void testResolveRefreshIntervalUsesDefault() { + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS, PartitionPlanKafkaConfig.resolveRefreshIntervalMs(new Properties(), PROP_PREFIX)); + } + + @Test + public void testResolveConsumerPollTimeoutUsesDefault() { + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(new Properties(), PROP_PREFIX)); + } + + @Test + public void testResolveConsumerPollTimeoutUsesOverride() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, "1000"); + assertEquals(1000, PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, PROP_PREFIX)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java new file mode 100644 index 0000000000..466e5463c5 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanRequestValidatorTest { + @Test + public void testValidateOnboardPluginAcceptsNonEmptyServices() { + Map services = Map.of( + "dev_hive", ServiceAllowlistEntry.ofUsers("hive"), + "dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + + assertDoesNotThrow(() -> PartitionPlanRequestValidator.validateOnboardPlugin( + new OnboardPlugin("hiveServer2", 3, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsNullRequest() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(null)); + } + + @Test + public void testValidateOnboardPluginRejectsMissingServices() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1))); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testValidateOnboardPluginRejectsEmptyServicesMap() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin( + new OnboardPlugin("trino", 2, 1, Collections.emptyMap()))); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testValidateOnboardPluginRejectsBlankPluginId() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("", 2, 1))); + } + + @Test + public void testValidateOnboardPluginRejectsNonPositivePartitionCount() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 0, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsBlankServiceName() { + Map services = new LinkedHashMap<>(); + services.put(" ", ServiceAllowlistEntry.ofUsers("hive")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("hiveServer2", 2, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsServiceWithoutAllowedUsers() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of(" "))); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1, services))); + } + + @Test + public void testValidateUpdatePluginAcceptsAddServicesOnly() { + UpdatePlugin update = new UpdatePlugin(1, null, Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")), null, null); + + assertDoesNotThrow(() -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", update)); + } + + @Test + public void testValidateUpdatePluginRejectsZeroAdditionalPartitions() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null))); + } + + @Test + public void testValidateUpdatePluginRejectsWithoutMutationDelta() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, null, null, null, null))); + } + + @Test + public void testValidateUpdatePluginRejectsBlankRemoveServiceName() { + UpdatePlugin update = new UpdatePlugin(1, null, null, null, List.of(" ")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", update)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java new file mode 100644 index 0000000000..5e270e62cf --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanServiceMutationTest { + private static final String TOPIC = "ranger_audits"; + + private PartitionPlan initialPlan; + + @BeforeEach + public void setUp() { + initialPlan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testOnboardPluginPublishesNextVersion() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, trinoServices()), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(2, registry.getPlan().getVersion()); + assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); + assertEquals("trino", result.getServices().get("dev_trino").getPluginId()); + assertEquals(1, registry.getWriteCount()); + assertEquals(result, PartitionPlanHolder.getInstance().getPlan()); + } + + @Test + public void testOnboardPluginWithMultipleServices() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + + PartitionPlan result = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals("trino", result.getServices().get("dev_hive").getPluginId()); + assertEquals("trino", result.getServices().get("dev_hive2").getPluginId()); + } + + @Test + public void testUpdatePluginAddsAndRemovesServices() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + PartitionPlan onboarded = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops"); + + UpdatePlugin update = new UpdatePlugin(onboarded.getVersion(), null, Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")), null, List.of("dev_hive2")); + PartitionPlan updated = service.updatePlugin("trino", update, "ops"); + + assertEquals(3, updated.getVersion()); + assertTrue(updated.getServices().containsKey("dev_hive")); + assertTrue(updated.getServices().containsKey("dev_hive3")); + assertFalse(updated.getServices().containsKey("dev_hive2")); + } + + @Test + public void testUpdatePluginScalesViaAdditionalPartitions() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.updatePlugin("hiveServer2", new UpdatePlugin(1, 2, null, null, null), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(17, result.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16), result.getPlugins().get("hiveServer2").getPartitions()); + } + + @Test + public void testOnboardPluginRejectsEmptyServicesMap() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, Collections.emptyMap()), "ops")); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testOnboardPluginRejectsMissingServices() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1), "ops")); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testOnboardPluginRejectsBlankPluginId() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin(" ", 3, 1), "ops")); + + assertTrue(error.getMessage().contains("pluginId")); + } + + @Test + public void testUpdatePluginRejectsInvalidAdditionalPartitions() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.updatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null), "ops")); + + assertTrue(error.getMessage().contains("additionalPartitions")); + } + + @Test + public void testOnboardPluginRejectsServiceWithoutAllowedUsers() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of())); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops")); + + assertTrue(error.getMessage().contains("allowedUsers")); + } + + @Test + public void testStaleExpectedVersionReturnsConflict() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 99, trinoServices()), "ops")); + + assertEquals(initialPlan, conflict.getCurrentPlan()); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testConflictWhenPeerPublishedBeforeWrite() { + PartitionPlan peerPlan = initialPlan.toBuilder().version(2).updatedBy("peer").build(); + MutableRegistry registry = new MutableRegistry(initialPlan) { + private final AtomicInteger reads = new AtomicInteger(); + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + if (reads.incrementAndGet() >= 2) { + return peerPlan; + } + return super.readPlan(auditTopicKey); + } + }; + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, trinoServices()), "ops")); + + assertEquals(peerPlan, conflict.getCurrentPlan()); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testTopicGrowFailureSurfacesAsPlanException() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new FailingAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 12, 1, trinoServices()), "ops")); + + assertTrue(error.getMessage().contains("grow audit topic")); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testDuplicateOnboardReturnsCurrentPlanWithoutWrite() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + OnboardPlugin request = new OnboardPlugin("trino", 3, 1, services); + + PartitionPlan first = service.onboardPlugin(request, "ops"); + assertEquals(1, registry.getWriteCount()); + + OnboardPlugin retry = new OnboardPlugin("trino", 3, first.getVersion(), services); + PartitionPlan second = service.onboardPlugin(retry, "ops"); + + assertEquals(first, second); + assertEquals(2, second.getVersion()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testUpdatePluginStillAppendsOnRepeatScale() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + service.updatePlugin("hiveServer2", new UpdatePlugin(1, 2, null, null, null), "ops"); + assertEquals(1, registry.getWriteCount()); + + PartitionPlan second = service.updatePlugin("hiveServer2", new UpdatePlugin(2, 2, null, null, null), "ops"); + + assertEquals(3, second.getVersion()); + assertEquals(2, registry.getWriteCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17, 18), second.getPlugins().get("hiveServer2").getPartitions()); + } + + private static Map trinoServices() { + return Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + } + + private static PartitionPlanService service(MutableRegistry registry, KafkaAuditTopicPartitionGrower topicGrower) { + return new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new FixedPartitionPlanRegistryFactory(registry), topicGrower); + } + + private static Properties enabledConfig() { + Properties props = new Properties(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, TOPIC); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } + + private static final class FixedPartitionPlanRegistryFactory extends PartitionPlanRegistryFactory { + private final PartitionPlanRegistry registry; + + private FixedPartitionPlanRegistryFactory(PartitionPlanRegistry registry) { + this.registry = registry; + } + + @Override + public PartitionPlanRegistry open(Properties props, String propPrefix) { + return registry; + } + } + + private static final class NoOpAuditTopicPartitionGrower extends KafkaAuditTopicPartitionGrower { + @Override + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + } + } + + private static final class FailingAuditTopicPartitionGrower extends KafkaAuditTopicPartitionGrower { + @Override + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + throw new RuntimeException("kafka down"); + } + } + + private static class MutableRegistry implements PartitionPlanRegistry { + private PartitionPlan plan; + private int writeCount; + + private MutableRegistry(PartitionPlan plan) { + this.plan = plan; + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + return plan; + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan newPlan) { + plan = newPlan; + writeCount++; + } + + @Override + public void close() { + } + + private PartitionPlan getPlan() { + return plan; + } + + private int getWriteCount() { + return writeCount; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java new file mode 100644 index 0000000000..254c597b92 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanServiceTest { + private static final String TOPIC = "ranger_audits"; + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testDynamicDisabledWhenFlagOff() { + PartitionPlanService service = new PartitionPlanService(disabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertFalse(service.isDynamicPartitionPlanEnabled()); + } + + @Test + public void testDynamicEnabledWhenFlagOn() { + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertTrue(service.isDynamicPartitionPlanEnabled()); + } + + @Test + public void testGetFromMemoryReturnsInstalledPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 3)); + PartitionPlanHolder.getInstance().install(plan, 6); + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + + PartitionPlan loaded = service.getPartitionPlan(); + + assertEquals(plan, loaded); + assertEquals(plan.toJson(), service.getPartitionPlan().toJson()); + } + + @Test + public void testGetFromMemoryFailsWhenPlanNotLoaded() { + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertThrows(PartitionPlanException.class, () -> service.getPartitionPlan()); + } + + private static Properties disabledConfig() { + Properties props = baseConfig(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "false"); + return props; + } + + private static Properties enabledConfig() { + Properties props = baseConfig(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } + + private static Properties baseConfig() { + Properties props = new Properties(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, TOPIC); + return props; + } + + private static final class NoOpPartitionPlanRegistryFactory extends PartitionPlanRegistryFactory { + @Override + public PartitionPlanRegistry open(Properties props, String propPrefix) { + return null; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java new file mode 100644 index 0000000000..09b264ce26 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class PartitionPlanUpdateApplierTest { + private static final String AUDIT_TOPIC = "ranger_audits"; + private static final String PLAN_TOPIC = "ranger_audit_partition_plan"; + private static final int KAFKA_PARTITIONS = 6; + + private PartitionPlanHolder holder; + private PartitionPlanUpdateApplier applier; + + @BeforeEach + public void setUp() { + holder = PartitionPlanHolder.getInstance(); + holder.resetForTests(); + holder.install(planWithVersion(1), KAFKA_PARTITIONS); + applier = new PartitionPlanUpdateApplier(new Properties(), AUDIT_TOPIC, holder, () -> KAFKA_PARTITIONS); + } + + @AfterEach + public void tearDown() { + holder.resetForTests(); + } + + @Test + public void testIgnoresWrongRecordKey() { + applier.applyRecordIfNewer(record("other-topic", planWithVersion(2).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testIgnoresSameVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(1).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testIgnoresOlderVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(0).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testInstallsNewerVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(2).toJson())); + + assertEquals(2, holder.getLastInstalledVersion()); + assertEquals(2, holder.getPlan().getVersion()); + } + + @Test + public void testIgnoresInvalidJson() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, "{not-json")); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testStartupSyncCanCatchPlanPublishedBeforeWatcherThread() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(2).toJson())); + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(3).toJson())); + + assertEquals(3, holder.getLastInstalledVersion()); + } + + private static PartitionPlan planWithVersion(int version) { + return PartitionPlan.builder() + .topic(AUDIT_TOPIC) + .version(version) + .topicPartitionCount(KAFKA_PARTITIONS) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .build(); + } + + private static ConsumerRecord record(String key, String value) { + return new ConsumerRecord<>(PLAN_TOPIC, 0, 0L, key, value); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java new file mode 100644 index 0000000000..83f732b885 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class PartitionPlanValidatorTest { + @Test + public void testValidateAcceptsWellFormedPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs"}, 2, 2)); + assertDoesNotThrow(() -> PartitionPlanValidator.validate(plan, 4)); + } + + @Test + public void testValidateRejectsDuplicatePartitionIds() { + Map plugins = new LinkedHashMap<>(); + plugins.put("hdfs", PluginPartitionAssignment.of(0, 1)); + PartitionPlan plan = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(3).plugins(plugins).buffer(PluginPartitionAssignment.of(1, 2)).build(); + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validate(plan)); + } + + @Test + public void testValidateRejectsKafkaPartitionMismatch() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs"}, 2, 2)); + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validate(plan, 10)); + } + + @Test + public void testValidateAppendOnlyRejectsReshuffle() { + PartitionPlan current = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(6).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5)).buffer(PluginPartitionAssignment.empty()).build(); + + Map reshuffled = new LinkedHashMap<>(); + reshuffled.put("hdfs", PluginPartitionAssignment.of(0, 1, 2, 3)); + reshuffled.put("hiveServer2", PluginPartitionAssignment.of(4, 5)); + PartitionPlan proposed = PartitionPlan.builder().topic("ranger_audits").version(2).topicPartitionCount(6).plugins(reshuffled).buffer(PluginPartitionAssignment.empty()).build(); + + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validateAppendOnly(current, proposed)); + } + + @Test + public void testValidateAppendOnlyAcceptsTailGrowth() { + PartitionPlan current = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(6).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5)).buffer(PluginPartitionAssignment.empty()).build(); + PartitionPlan proposed = PartitionPlan.builder().topic("ranger_audits").version(2).topicPartitionCount(9).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5, 6, 7, 8)).buffer(PluginPartitionAssignment.empty()).build(); + assertDoesNotThrow(() -> PartitionPlanValidator.validateAppendOnly(current, proposed)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java new file mode 100644 index 0000000000..66298e93f3 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_SUFFIX_ALLOWED_USERS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ServiceAllowlistBootstrapTest { + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testLoadFromPropertiesParsesServiceRepos() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_ozone" + PROP_SUFFIX_ALLOWED_USERS, "om, ozone"); + + Map services = ServiceAllowlistBootstrap.loadAllowlistsFromProperties(props); + + assertEquals(2, services.size()); + assertIterableEquals(List.of("hive"), services.get("dev_hive").getAllowedUsers()); + assertIterableEquals(List.of("om", "ozone"), services.get("dev_ozone").getAllowedUsers()); + } + + @Test + public void testEnrichServicesFromXmlIfMissingMergesInMemoryOnly() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(3) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .build(); + + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + + PartitionPlan enriched = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + + assertEquals(3, enriched.getVersion()); + assertNotNull(enriched.getServices().get("dev_hive")); + PartitionPlanHolder.getInstance().install(enriched, 6); + assertIterableEquals(List.of("hive"), PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + } + + @Test + public void testMergeSkippedWhenPlanAlreadyHasPartialServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(services) + .build(); + + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + + PartitionPlan merged = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + + assertEquals(plan, merged); + assertEquals(1, merged.getServices().size()); + assertNotNull(merged.getServices().get("dev_hdfs")); + assertTrue(merged.getServices().get("dev_hive") == null); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java new file mode 100644 index 0000000000..755a2a7835 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ServiceAllowlistResolverTest { + private static final Map> STATIC = Map.of( + "dev_hive", Set.of("hive"), + "dev_hdfs", Set.of("hdfs", "nn")); + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testRejectsBlankServiceOrUser() { + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("", "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser(null, "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testUsesStaticXmlWhenDynamicDisabled() { + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hive", false, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hdfs", false, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testUsesRegistryWhenRepoOnboarded() { + installPartialPlan("dev_hdfs", "hdfs"); + + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "hdfs", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "nn", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testFallsBackToStaticXmlWhenRepoNotInRegistry() { + installPartialPlan("dev_hdfs", "hdfs"); + + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_trino", "trino", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testDeniesWhenUserNotInRegistryAllowlist() { + Map services = Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder holder = PartitionPlanHolder.getInstance(); + holder.install(plan, 6); + + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "nn", true, holder, STATIC)); + } + + private static void installPartialPlan(String repo, String user) { + Map services = new LinkedHashMap<>(); + services.put(repo, ServiceAllowlistEntry.ofUsers(user)); + PartitionPlanHolder.getInstance().install(basePlanBuilder().services(services).build(), 6); + } + + private static PartitionPlan.Builder basePlanBuilder() { + return PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java new file mode 100644 index 0000000000..b4fb2aa9bb --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrap; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrapConfig; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanJsonTest { + @Test + public void testRoundTripPreservesPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs", "hiveServer2"}, 3, 9)); + PartitionPlan parsed = PartitionPlan.fromJson(plan.toJson()); + + assertEquals(plan.getTopic(), parsed.getTopic()); + assertEquals(plan.getVersion(), parsed.getVersion()); + assertEquals(plan.getTopicPartitionCount(), parsed.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2), parsed.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), parsed.getBuffer().getPartitions()); + } + + @Test + public void testRoundTripPreservesServices() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive"))) + .build(); + PartitionPlan parsed = PartitionPlan.fromJson(plan.toJson()); + + assertIterableEquals(List.of("hive"), parsed.getServices().get("dev_hive").getAllowedUsers()); + } + + @Test + public void testSameContentAsIgnoresVersionAndMetadata() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive"))) + .build(); + PartitionPlan withMetadata = plan.toBuilder().version(99).updatedAt("later").updatedBy("other").build(); + + assertTrue(plan.sameContentAs(withMetadata)); + } + + @Test + public void testFromJsonRejectsInvalidPlan() { + assertThrows(PartitionPlanException.class, () -> PartitionPlan.fromJson("{\"topic\":\"ranger_audits\",\"version\":1}")); + } +}