This is an automated email from the ASF dual-hosted git repository. nickallen pushed a commit to branch feature/METRON-2088-support-hdp-3.1 in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/feature/METRON-2088-support-hdp-3.1 by this push: new 939c4ea METRON-2265 Update Kerberos settings (merrimanr via nickwallen) closes apache/metron#1519 939c4ea is described below commit 939c4ea6b43d73c086dddbb86078d8d6878fc83f Author: merrimanr <merrim...@gmail.com> AuthorDate: Thu Oct 3 13:26:07 2019 -0400 METRON-2265 Update Kerberos settings (merrimanr via nickwallen) closes apache/metron#1519 --- metron-deployment/Kerberos-manual-setup.md | 65 +++++++++++++++------- .../roles/sensor-stubs/templates/start-bro-stub | 1 + .../roles/sensor-stubs/templates/start-snort-stub | 1 + .../roles/sensor-stubs/templates/start-yaf-stub | 1 + .../CURRENT/package/scripts/params/params_linux.py | 2 +- .../METRON/CURRENT/package/templates/metron.j2 | 12 ++-- .../org/apache/metron/rest/config/KafkaConfig.java | 2 +- .../apache/metron/rest/config/KafkaConfigTest.java | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 22 +++++--- .../metron-common/src/main/scripts/stellar | 9 +++ metron-platform/metron-data-management/pom.xml | 7 +++ .../metron-elasticsearch-storm/pom.xml | 4 ++ .../AbstractLuceneMetaAlertUpdateDaoTest.java | 5 +- .../apache/metron/management/KafkaFunctions.java | 11 +++- .../metron-parsing/metron-parsing-storm/pom.xml | 12 ++++ 15 files changed, 114 insertions(+), 42 deletions(-) diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index d50da34..aeedd95 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -70,7 +70,6 @@ Setup 1. Create the `metron` user's home directory in HDFS. ``` - sudo -u hdfs hdfs dfs -mkdir /user/metron sudo -u hdfs hdfs dfs -chown metron:hdfs /user/metron sudo -u hdfs hdfs dfs -chmod 770 /user/metron ``` @@ -84,6 +83,38 @@ Setup a KDC yum -y install krb5-server krb5-libs krb5-workstation ``` +1. Uncomment all commented lines in `/etc/krb5.conf`. The contents of `/etc/krb5.conf` should look like: + + ``` + # Configuration snippets may be placed in this directory as well + includedir /etc/krb5.conf.d/ + + [logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + pkinit_anchors = /etc/pki/tls/certs/ca-bundle.crt + default_realm = EXAMPLE.COM + default_ccache_name = KEYRING:persistent:%{uid} + + [realms] + EXAMPLE.COM = { + kdc = node1 + admin_server = node1 + } + + [domain_realm] + .example.com = EXAMPLE.COM + example.com = EXAMPLE.COM + ``` + 1. Define the current host as the KDC. ``` @@ -92,7 +123,7 @@ Setup a KDC cp -f /etc/krb5.conf /var/lib/ambari-server/resources/scripts ``` -1. Ensure that the KDC can issue renewable tickets. This may be necessary on a real cluster, but should not be on a [single VM](development/centos6/README.md). +1. Ensure that the KDC can issue renewable tickets. Edit `/var/kerberos/krb5kdc/kdc.conf` and ensure the following is added to the `realm` section @@ -109,14 +140,14 @@ Setup a KDC 1. Start the KDC and ensure that it starts on boot. ``` - /etc/rc.d/init.d/krb5kdc start + krb5kdc chkconfig krb5kdc on ``` 1. Start the Kerberos Admin service and ensure that it starts on boot. ``` - /etc/rc.d/init.d/kadmin start + kadmind chkconfig kadmin on ``` @@ -203,6 +234,7 @@ Enable Kerberos ``` kadmin.local -q "ktadd -k metron.headless.keytab met...@example.com" + mkdir /etc/security/keytabs cp metron.headless.keytab /etc/security/keytabs chown metron:hadoop /etc/security/keytabs/metron.headless.keytab chmod 440 /etc/security/keytabs/metron.headless.keytab @@ -417,9 +449,12 @@ Push Data 1. Push some sample data to one of the parser topics. E.g for Bro we took raw data from [metron/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput](../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput) ``` - cat sample-bro.txt | ${KAFKA_HOME}/kafka-broker/bin/kafka-console-producer.sh \ + source /etc/default/metron + kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME + export KAFKA_OPTS=$CLIENT_JAAS_ARG + cat sample-bro.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \ --broker-list ${BROKERLIST} \ - --security-protocol SASL_PLAINTEXT \ + --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \ --topic bro ``` @@ -494,7 +529,7 @@ More notes can be found in [metron/metron-sensors/README.md](../metron-sensors/R ``` cat sample-yaf.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \ --broker-list ${BROKERLIST} \ - --security-protocol PLAINTEXTSASL \ + --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \ --topic yaf ``` @@ -502,18 +537,10 @@ cat sample-yaf.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \ ``` ${KAFKA_HOME}/bin/kafka-console-consumer.sh \ - --zookeeper ${ZOOKEEPER} \ - --security-protocol PLAINTEXTSASL \ - --topic yaf -``` - -#### Modify the sensor-stubs to send logs via SASL -``` -sed -i 's/node1:6667 --topic/node1:6667 --security-protocol PLAINTEXTSASL --topic/' /opt/sensor-stubs/bin/start-*-stub -for sensorstub in bro snort; do - service sensor-stubs stop ${sensorstub}; - service sensor-stubs start ${sensorstub}; -done + --bootstrap-server ${BROKERLIST} \ + --consumer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \ + --topic yaf \ + --consumer-property group.id=<authorized group> ``` #### Model as a Service on Kerberos diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub index f45f78e..493b752 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub @@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} if [ ${SECURITY_ENABLED,,} == 'true' ]; then echo "Security enabled" kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME + export KAFKA_OPTS=$CLIENT_JAAS_ARG fi while true; do diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub index b37bafc..e3deb33 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub @@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} if [ ${SECURITY_ENABLED,,} == 'true' ]; then echo "Security enabled" kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME + export KAFKA_OPTS=$CLIENT_JAAS_ARG fi while true; do diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub index 22b3fbe..8efbb02 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub @@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} if [ ${SECURITY_ENABLED,,} == 'true' ]; then echo "Security enabled" kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME + export KAFKA_OPTS=$CLIENT_JAAS_ARG fi while true; do diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 2e9bfac..b7fbcf4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -263,7 +263,7 @@ topology_auto_credentials = config['configurations']['storm-site'].get('nimbus.c topology_auto_credentials_double_quotes = str(topology_auto_credentials).replace("'", '"') if security_enabled: - hostname_lowercase = config['hostname'].lower() + hostname_lowercase = hostname.lower() metron_principal_name = status_params.metron_principal_name metron_keytab_path = status_params.metron_keytab_path kinit_path_local = status_params.kinit_path_local diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 5c43bbd..71ecf0f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -48,16 +48,19 @@ METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}" METRON_USER_ROLE="{{metron_user_role}}" METRON_ADMIN_ROLE="{{metron_admin_role}}" -ZOOKEEPER="{{zookeeper_quorum}}" -BROKERLIST="{{kafka_brokers}}" -HADOOP_CONF_DIR="/etc/hadoop/conf/" -STORM_REST_URL="{{storm_rest_addr}}" +#Security SECURITY_ENABLED={{security_enabled|lower}} {% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}" {% endif %} {% if metron_keytab_path is defined %}METRON_SERVICE_KEYTAB="{{metron_keytab_path}}" {% endif %} KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}" +CLIENT_JAAS_ARG="{{client_jaas_arg}}" + +ZOOKEEPER="{{zookeeper_quorum}}" +BROKERLIST="{{kafka_brokers}}" +HADOOP_CONF_DIR="/etc/hadoop/conf/" +STORM_REST_URL="{{storm_rest_addr}}" PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config" METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}" PCAP_BASE_PATH="{{pcap_base_path}}" @@ -68,4 +71,3 @@ PCAP_YARN_QUEUE="{{pcap_yarn_queue}}" PCAP_FINALIZER_THREADPOOL_SIZE="{{pcap_finalizer_threadpool_size}}" STORM_STATUS_CACHE_MAX_SIZE="{{storm_status_cache_max_size}}" STORM_STATUS_CACHE_TIMEOUT_SECONDS="{{storm_status_cache_timeout_seconds}}" - diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java index 52a7dbb..43eb943 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java @@ -99,7 +99,7 @@ public class KafkaConfig { producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); producerConfig.put(ProducerConfig.ACKS_CONFIG, "1"); if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) { - producerConfig.put("security.protocol", SecurityProtocol.SASL_PLAINTEXT); + producerConfig.put("security.protocol", KafkaUtils.INSTANCE.normalizeProtocol(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY))); } return producerConfig; } diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/KafkaConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/KafkaConfigTest.java index dab924f..4694917 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/KafkaConfigTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/KafkaConfigTest.java @@ -67,7 +67,7 @@ public class KafkaConfigTest { when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(true); when(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)).thenReturn("kafka security protocol"); - producerProperties = kafkaConfig.consumerProperties(); + producerProperties = kafkaConfig.producerProperties(); assertEquals("kafka security protocol", producerProperties.get("security.protocol")); } diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index 93a05a0..31426f9 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -135,15 +135,19 @@ </filter> </filters> <relocations> - <!-- The REST API has conflicts with these packages --> - <relocation> - <pattern>io.netty</pattern> - <shadedPattern>org.apache.metron.io.netty</shadedPattern> - </relocation> - <relocation> - <pattern>com.google.common</pattern> - <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern> - </relocation> + <!-- The REST API has conflicts with these packages --> + <relocation> + <pattern>io.netty</pattern> + <shadedPattern>org.apache.metron.io.netty</shadedPattern> + </relocation> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.http</pattern> + <shadedPattern>org.apache.metron.http.elasticsearch</shadedPattern> + </relocation> </relocations> <artifactSet> <excludes> diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index 6f9418b..bd4ab58 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -36,6 +36,15 @@ fi # treat unset vars as an error; METRON_HOME set -u +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + echo "Security enabled" + kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME + export KAFKA_OPTS=$CLIENT_JAAS_ARG +fi + export METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}" export HBASE_CONFIGS=$(hbase classpath) export STELLAR_LIB=$(find $METRON_HOME/lib/ -name stellar-common*.jar) diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 07f98cf..c9b4301 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -417,6 +417,13 @@ <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern> </relocation> <relocation> + <!-- need to also relocate Guava's `thirdparty` packages. otherwise, the class + `thirdparty.publicsuffix.PublicSuffixPatterns` cannot be found after + relocation of Guava's `com.google.commmon` packages. --> + <pattern>com.google.thirdparty</pattern> + <shadedPattern>org.apache.metron.guava.thirdparty.${guava_version}</shadedPattern> + </relocation> + <relocation> <pattern>org.apache.http</pattern> <shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern> </relocation> diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml index 1ba1e27..9fccf68 100644 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml @@ -167,6 +167,10 @@ <pattern>com.fasterxml.jackson</pattern> <shadedPattern>org.apache.metron.jackson</shadedPattern> </relocation> + <relocation> + <pattern>org.apache.http</pattern> + <shadedPattern>org.apache.metron.http.elasticsearch</shadedPattern> + </relocation> </relocations> <artifactSet> <excludes> diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java index ec241e4..58e15ea 100644 --- a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java +++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java @@ -46,7 +46,6 @@ import java.util.UUID; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; -import org.apache.commons.math.util.MathUtils; import org.apache.metron.common.Constants; import org.apache.metron.common.Constants.Fields; import org.apache.metron.indexing.dao.IndexDao; @@ -846,11 +845,11 @@ public class AbstractLuceneMetaAlertUpdateDaoTest { Object value = entry.getValue(); Object actualValue = actual.getDocument().get(entry.getKey()); if (value instanceof Float) { - if (!MathUtils.equals((Float) value, (Float) actualValue, EPS)) { + if (!(Math.abs((Float) value - (Float) actualValue) < EPS)) { return false; } } else if (value instanceof Double) { - if (!MathUtils.equals((Double) value, (Double) actualValue, EPS)) { + if (!(Math.abs((Double) value - (Double) actualValue) < EPS)) { return false; } } else { diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java index 78026fb..f981c71 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java @@ -175,7 +175,9 @@ public class KafkaFunctions { namespace = "KAFKA", name = "GET", description = "Retrieves messages from a Kafka topic. Subsequent calls will" + - "continue retrieving messages sequentially from the original offset.", + "continue retrieving messages sequentially from the original offset. When Kerberos is enabled, the " + + "'security.protocol' config key must be set to 'SASL_PLAINTEXT' and the 'group.id' config key must be " + + "set to an authorized group.", params = { "topic - The name of the Kafka topic", "count - The number of Kafka messages to retrieve", @@ -291,7 +293,9 @@ public class KafkaFunctions { @Stellar( namespace = "KAFKA", name = "TAIL", - description = "Tails messages from a Kafka topic always starting with the most recently received message.", + description = "Tails messages from a Kafka topic always starting with the most recently received message. When " + + "Kerberos is enabled, the 'security.protocol' config key must be set to 'SASL_PLAINTEXT' and the " + + "'group.id' config key must be set to an authorized group.", params = { "topic - The name of the Kafka topic", "count - The number of Kafka messages to retrieve", @@ -403,7 +407,8 @@ public class KafkaFunctions { @Stellar( namespace = "KAFKA", name = "PUT", - description = "Sends messages to a Kafka topic. ", + description = "Sends messages to a Kafka topic. When Kerberos is enabled, the 'security.protocol' config key " + + "must be set to 'SASL_PLAINTEXT' and the 'group.id' config key must be set to an authorized group.", params = { "topic - The name of the Kafka topic.", "messages - A list of messages to write.", diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml index c983be5..a94ad77 100644 --- a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml +++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml @@ -28,6 +28,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <guava_version>${global_hbase_guava_version}</guava_version> </properties> <dependencies> <!-- Dependencies needed preferentially --> @@ -218,6 +219,17 @@ <pattern>com.fasterxml.jackson</pattern> <shadedPattern>org.apache.metron.jackson</shadedPattern> </relocation> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern> + </relocation> + <relocation> + <!-- need to also relocate Guava's `thirdparty` packages. otherwise, the class + `thirdparty.publicsuffix.PublicSuffixPatterns` cannot be found after + relocation of Guava's `com.google.commmon` packages. --> + <pattern>com.google.thirdparty</pattern> + <shadedPattern>org.apache.metron.guava.thirdparty.${guava_version}</shadedPattern> + </relocation> </relocations> <artifactSet> <excludes>