wrapper alert engine as one application of eagle server hdfs audit log + alert engine end-2-end
https://issues.apache.org/jira/browse/EAGLE-481 Author: @yonzhang <yonzhang2...@apache.org> Closes: #392 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/000b7460 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/000b7460 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/000b7460 Branch: refs/heads/master Commit: 000b746041ec07774e4ed5d5562f9c66307f1c20 Parents: 36f9a84 Author: yonzhang <yonzhang2...@gmail.com> Authored: Thu Aug 25 21:39:53 2016 -0700 Committer: yonzhang <yonzhang2...@gmail.com> Committed: Thu Aug 25 21:39:53 2016 -0700 ---------------------------------------------------------------------- .../eagle-alert-parent/eagle-alert-app/pom.xml | 35 ++++ .../eagle/alert/app/AlertUnitTopologyApp.java | 39 ++++ .../alert/app/AlertUnitTopologyAppProvider.java | 29 +++ ...e.alert.app.AlertUnitTopologyAppProvider.xml | 135 +++++++++++++ ...org.apache.eagle.app.spi.ApplicationProvider | 18 ++ .../src/main/resources/application.conf | 60 ++++++ .../alert/AlertPolicyValidateProvider.java | 36 ---- .../eagle/alert/config/ZKConfigBuilder.java | 8 +- .../scheme/JsonStringStreamNameSelector.java | 6 +- .../alert/engine/spout/CorrelationSpout.java | 4 +- .../src/main/resources/application.conf | 19 +- .../alert/engine/e2e/CoordinatorClient.java | 6 +- eagle-core/eagle-alert-parent/pom.xml | 9 +- .../eagle/app/service/ApplicationContext.java | 6 +- .../src/main/resources/log4j.properties | 2 +- .../eagle-security-hdfs-auditlog/README.md | 191 +++++++++++++++++++ ...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 26 ++- .../src/main/resources/scripts.txt | 22 +++ eagle-server/pom.xml | 7 + 19 files changed, 581 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml new file mode 100644 index 0000000..768709f --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-alert-parent</artifactId> + <version>0.5.0-incubating-SNAPSHOT</version> + </parent> + <artifactId>eagle-alert-app</artifactId> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-app-base</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java new file mode 100644 index 0000000..a80c3b8 --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.alert.app;import backtype.storm.generated.StormTopology; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.apache.eagle.alert.engine.UnitTopologyMain; +import org.apache.eagle.app.StormApplication; +import org.apache.eagle.app.environment.impl.StormEnvironment; + +/** + * since 8/25/16. + */ +public class AlertUnitTopologyApp extends StormApplication { + @Override + public StormTopology execute(Config config, StormEnvironment environment) { + return UnitTopologyMain.createTopology(config); + } + + public static void main(String[] args){ + Config config = ConfigFactory.load(); + AlertUnitTopologyApp app = new AlertUnitTopologyApp(); + app.run(config); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java new file mode 100644 index 0000000..39a4583 --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.alert.app; +import org.apache.eagle.app.spi.AbstractApplicationProvider; + +/** + * since 8/25/16. + */ +public class AlertUnitTopologyAppProvider extends AbstractApplicationProvider<AlertUnitTopologyApp> { + @Override + public AlertUnitTopologyApp getApplication() { + return new AlertUnitTopologyApp(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml new file mode 100644 index 0000000..498cb8d --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<application> + <type>AlertUnitTopologyApp</type> + <name>Alert Unit Topology Application</name> + <version>0.5.0-incubating</version> + <appClass>org.apache.eagle.alert.app.AlertUnitTopologyApp</appClass> + <viewPath>/apps/alert</viewPath> + <configuration> + <!-- alert topology sizing parameters --> + <property> + <name>topology.numOfTotalWorkers</name> + <displayName>topology.numOfTotalWorkers</displayName> + <value>2</value> + <description>number of total storm workers</description> + </property> + <property> + <name>topology.numOfSpoutTasks</name> + <displayName>topology.numOfSpoutTasks</displayName> + <value>1</value> + <description>number of spout tasks</description> + </property> + <property> + <name>topology.numOfRouterBolts</name> + <displayName>topology.numOfRouterBolts</displayName> + <value>4</value> + <description>number of router tasks</description> + </property> + <property> + <name>topology.numOfAlertBolts</name> + <displayName>topology.numOfAlertBolts</displayName> + <value>10</value> + <description>number of alert tasks</description> + </property> + <property> + <name>topology.numOfPublishTasks</name> + <displayName>topology.numOfPublishTasks</displayName> + <value>1</value> + <description>number of publish tasks</description> + </property> + <property> + <name>topology.messageTimeoutSecs</name> + <displayName>topology.messageTimeoutSecs</displayName> + <value>3600</value> + <description>number of tuple timeout in seconds</description> + </property> + + <!-- alert spout configuration --> + <property> + <name>spout.kafkaBrokerZkQuorum</name> + <displayName>spout.kafkaBrokerZkQuorum</displayName> + <value>server.eagle.apache.org:2181</value> + <description>zookeeper quorum for spout to consume data</description> + </property> + <property> + <name>spout.kafkaBrokerZkBasePath</name> + <displayName>spout.kafkaBrokerZkBasePath</displayName> + <value>/brokers</value> + <description>zk znode path for kafka brokers</description> + </property> + <property> + <name>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</name> + <displayName>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</displayName> + <value>true</value> + <description>same zookeeper for kafka server and kafka consumer(Storm-Kafka)</description> + </property> + <property> + <name>spout.stormKafkaTransactionZkPath</name> + <displayName>spout.stormKafkaTransactionZkPath</displayName> + <value>/consumers</value> + <description>zk path for storm kafka transaction</description> + </property> + <property> + <name>spout.stormKafkaEagleConsumer</name> + <displayName>spout.stormKafkaEagleConsumer</displayName> + <value>eagle_consumer</value> + <description>zookeeper quorum for spout to consume data</description> + </property> + + <!-- zk config for alert engine --> + <property> + <name>zkConfig.zkQuorum</name> + <displayName>zkConfig.zkQuorum</displayName> + <value>server.eagle.apache.org:2181</value> + <description>zk quorum for alert engine</description> + </property> + <property> + <name>zkConfig.zkRoot</name> + <displayName>zkConfig.zkRoot</displayName> + <value>/alert</value> + <description>zk znode path for alert engine</description> + </property> + + <property> + <name>metadataService.context</name> + <displayName>metadataService.context</displayName> + <value>/rest</value> + <description>metadata service context path</description> + </property> + <property> + <name>metadataService.host</name> + <displayName>metadataService.host</displayName> + <value>localhost</value> + <description>metadata service host</description> + </property> + <property> + <name>metadataService.port</name> + <displayName>metadataService.port</displayName> + <value>9090</value> + <description>metadata service port</description> + </property> + </configuration> + <docs> + <install> + </install> + <uninstall> + </uninstall> + </docs> +</application> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider new file mode 100644 index 0000000..ceb079d --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.eagle.alert.app.AlertUnitTopologyAppProvider http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf new file mode 100644 index 0000000..1a25cfa --- /dev/null +++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + "appId" : "HBaseAuditLogApp", + "mode" : "LOCAL", + "siteId" : "testsite", + "topology" : { + "name" : "alertUnitTopology_1", + "numOfTotalWorkers" : 2, + "numOfSpoutTasks" : 1, + "numOfRouterBolts" : 4, + "numOfAlertBolts" : 10, + "numOfPublishTasks" : 1, + "messageTimeoutSecs": 3600, + "localMode" : "true" + }, + "spout" : { + "kafkaBrokerZkQuorum": "server.eagle.apache.org:2181", + "kafkaBrokerZkBasePath": "/kafka", + "stormKafkaUseSameZkQuorumWithKafkaBroker": true, + "stormKafkaTransactionZkQuorum": "", + "stormKafkaTransactionZkPath": "/consumers", + "stormKafkaEagleConsumer": "eagle_consumer" + }, + "zkConfig" : { + "zkQuorum" : "server.eagle.apache.org:2181", + "zkRoot" : "/alert" + }, + "metadataService": { + "context" : "/rest", + "host" : "localhost", + "port" : 9090 + }, + "metric":{ + "sink": { + // "kafka": { + // "topic": "alert_metric" + // "bootstrap.servers": "localhost:6667" + // } + // "stdout": {} + // "elasticsearch": { + // "hosts": ["localhost:9200"] + // "index": "alert_metric" + // "timestampField": "timestamp" + // } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java b/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java deleted file mode 100644 index 47ddcb9..0000000 --- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.alert; - -import java.util.List; - -import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.Module; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible=true) -@JsonIgnoreProperties(ignoreUnknown = true) -public abstract class AlertPolicyValidateProvider { - public String type; - - public abstract GenericServiceAPIResponseEntity<String> validate(); - - public abstract String PolicyType(); - - public abstract List<Module> BindingModules(); -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java index 9d77a58..d7b3232 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java @@ -29,10 +29,10 @@ public class ZKConfigBuilder { ZKConfig zkConfig = new ZKConfig(); zkConfig.zkQuorum = config.getString("zkConfig.zkQuorum"); zkConfig.zkRoot = config.getString("zkConfig.zkRoot"); - zkConfig.zkSessionTimeoutMs = config.getInt("zkConfig.zkSessionTimeoutMs"); - zkConfig.connectionTimeoutMs = config.getInt("zkConfig.connectionTimeoutMs"); - zkConfig.zkRetryTimes = config.getInt("zkConfig.zkRetryTimes"); - zkConfig.zkRetryInterval = config.getInt("zkConfig.zkRetryInterval"); + zkConfig.zkSessionTimeoutMs = config.hasPath("zkConfig.zkSessionTimeoutMs") ? config.getInt("zkConfig.zkSessionTimeoutMs") : 10000; + zkConfig.connectionTimeoutMs = config.hasPath("zkConfig.connectionTimeoutMs") ? config.getInt("zkConfig.connectionTimeoutMs") : 10000; + zkConfig.zkRetryTimes = config.hasPath("zkConfig.zkRetryTimes") ? config.getInt("zkConfig.zkRetryTimes") : 3; + zkConfig.zkRetryInterval = config.hasPath("zkConfig.zkRetryInterval") ? config.getInt("zkConfig.zkRetryInterval") : 3000; return zkConfig; } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java index 1182e3f..226dd84 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java @@ -33,9 +33,9 @@ import org.slf4j.LoggerFactory; */ public class JsonStringStreamNameSelector implements StreamNameSelector { private final static Logger LOG = LoggerFactory.getLogger(JsonStringStreamNameSelector.class); - private final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName"; - private final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName"; - private final static String STREAM_NAME_FORMAT = "streamNameFormat"; + public final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName"; + public final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName"; + public final static String STREAM_NAME_FORMAT = "streamNameFormat"; private String userProvidedStreamName; private String[] fieldNamesToInferStreamName; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java index f54d5cd..99c1fed 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java @@ -332,9 +332,9 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener spoutConfig.zkPort = utils.getZkPort(); } // transaction update interval - spoutConfig.stateUpdateIntervalMs = config.getLong("spout.stormKafkaStateUpdateIntervalMs"); + spoutConfig.stateUpdateIntervalMs = config.hasPath("spout.stormKafkaStateUpdateIntervalMs") ? config.getInt("spout.stormKafkaStateUpdateIntervalMs") : 2000; // Kafka fetch size - spoutConfig.fetchSizeBytes = config.getInt("spout.stormKafkaFetchSizeBytes"); + spoutConfig.fetchSizeBytes = config.hasPath("spout.stormKafkaFetchSizeBytes") ? config.getInt("spout.stormKafkaFetchSizeBytes") : 1048586; // "startOffsetTime" is for test usage, prod should not use this if (config.hasPath("spout.stormKafkaStartOffsetTime")) { spoutConfig.startOffsetTime = config.getInt("spout.stormKafkaStartOffsetTime"); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf index 7030e45..dac2f07 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf @@ -29,32 +29,17 @@ "stormKafkaUseSameZkQuorumWithKafkaBroker": true, "stormKafkaTransactionZkQuorum": "", "stormKafkaTransactionZkPath": "/consumers", - "stormKafkaEagleConsumer": "eagle_consumer", - "stormKafkaStateUpdateIntervalMs": 2000, - "stormKafkaFetchSizeBytes": 1048586, + "stormKafkaEagleConsumer": "eagle_consumer" }, "zkConfig" : { "zkQuorum" : "server.eagle.apache.org:2181", - "zkRoot" : "/alert", - "zkSessionTimeoutMs" : 10000, - "connectionTimeoutMs" : 10000, - "zkRetryTimes" : 3, - "zkRetryInterval" : 3000 - }, - "dynamicConfigSource" : { - "initDelayMillis": 3000, - "delayMillis" : 10000 + "zkRoot" : "/alert" }, "metadataService": { "context" : "/rest", "host" : "localhost", "port" : 9090 }, - "coordinatorService": { - "host": "localhost", - "port": 9090, - "context" : "/rest" - } "metric":{ "sink": { // "kafka": { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java index aebf3b5..0cf5115 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java @@ -41,9 +41,9 @@ public class CoordinatorClient implements Closeable { @SuppressWarnings("unused") private static final Logger LOG = LoggerFactory.getLogger(CoordinatorClient.class); - private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "coordinatorService.context"; - private static final String EAGLE_COORDINATOR_SERVICE_PORT = "coordinatorService.port"; - private static final String EAGLE_COORDINATOR_SERVICE_HOST = "coordinatorService.host"; + private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "metadataService.context"; + private static final String EAGLE_COORDINATOR_SERVICE_PORT = "metadataService.port"; + private static final String EAGLE_COORDINATOR_SERVICE_HOST = "metadataService.host"; private static final String COORDINATOR_SCHEDULE_API = "/coordinator/build"; private String host; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/pom.xml b/eagle-core/eagle-alert-parent/pom.xml index d5166cb..e2ba4d8 100644 --- a/eagle-core/eagle-alert-parent/pom.xml +++ b/eagle-core/eagle-alert-parent/pom.xml @@ -15,9 +15,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.eagle</groupId> @@ -33,5 +31,6 @@ <modules> <module>eagle-alert</module> <module>eagle-alert-service</module> - </modules> -</project> + <module>eagle-alert-app</module> + </modules> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java index 52eb628..1a80b78 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java @@ -102,9 +102,9 @@ public class ApplicationContext implements Serializable, ApplicationLifecycle { datasource.setTopic(kafkaCfg.getTopicId()); datasource.setSchemeCls(JsonScheme.class.getCanonicalName()); Tuple2StreamMetadata tuple2Stream = new Tuple2StreamMetadata(); - Set<String> activeStreamNames = new HashSet<>(); - activeStreamNames.add(streamDesc.getSchema().getStreamId()); - tuple2Stream.setActiveStreamNames(activeStreamNames); + Properties prop = new Properties(); + prop.put(JsonStringStreamNameSelector.USER_PROVIDED_STREAM_NAME_PROPERTY, streamDesc.getStreamId()); + tuple2Stream.setStreamNameSelectorProp(prop); tuple2Stream.setTimestampColumn("timestamp"); tuple2Stream.setStreamNameSelectorCls(JsonStringStreamNameSelector.class.getCanonicalName()); datasource.setCodec(tuple2Stream); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties index fb13ad5..d59ded6 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties +++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=DEBUG, stdout +log4j.rootLogger=INFO, stdout # standard output log4j.appender.stdout=org.apache.log4j.ConsoleAppender http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/README.md ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/README.md b/eagle-security/eagle-security-hdfs-auditlog/README.md new file mode 100644 index 0000000..8d2e97a --- /dev/null +++ b/eagle-security/eagle-security-hdfs-auditlog/README.md @@ -0,0 +1,191 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +Development in IDE + +## 1. Start eagle-server +In IDE, configure the following main class and program arguments + +org.apache.eagle.server.ServerMain server src/main/resources/configuration.yml + +## 2. Start alert engine + +### 2.1 Create new site + +http://localhost:9090/rest/sites POST +``` +{ +"siteId" : "testsite", +"siteName" :"testsite", +"description" : "test description", +"context" : {} +} +``` + +###n2.2 Create logic alert engine topology + +http://localhost:9090/rest/metadata/topologies POST +``` +{ + "name": "alertUnitTopology_1", + "numOfSpout": 1, + "numOfAlertBolt": 10, + "numOfGroupBolt": 4, + "spoutId": "alertEngineSpout", + "groupNodeIds": [ + "streamRouterBolt0", + "streamRouterBolt1", + "streamRouterBolt2", + "streamRouterBolt3" + ], + "alertBoltIds": [ + "alertBolt0", + "alertBolt1", + "alertBolt2", + "alertBolt3", + "alertBolt4", + "alertBolt5", + "alertBolt6", + "alertBolt7", + "alertBolt8", + "alertBolt9" + ], + "pubBoltId": "alertPublishBolt", + "spoutParallelism": 1, + "groupParallelism": 1, + "alertParallelism": 1 +} +``` + +### 2.3 Install alert engine application +Please reference eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml for +complete configuration. + +http://localhost:9090/rest/apps/install POST +``` +{ +"siteId" : "testsite", +"appType" : "AlertUnitTopologyApp", +"mode" : "LOCAL", +"configuration" : { + } +} +``` + +### 2.4 Run alert engine +Please use correct uuid + +http://localhost:9090/rest/apps/start POST +``` +{ +"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3", +"appId": "AlertUnitTopologyAppï¼testsite" +} +``` + +## 3 Start Hdfs audit log monitoring application + +### 3.1 Install HdfsAuditLog app + +http://localhost:9090/rest/apps/install POST +``` +{ +"siteId" : "testsite", +"appType" : "HdfsAuditLogApplication", +"mode" : "LOCAL", +"configuration" : { + "dataSourceConfig.topic" :"hdfs_audit_log"} +} +``` + +### 3.2 Start HdfsAuditLog app +Please use correct uuid + +http://localhost:9090/rest/apps/start POST +``` +{ +"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3", +"appId": "HdfsAuditLogApplicationï¼testsite" +} +``` + +## 4 Check +### 4.1 Check if alert data source is created +http://localhost:9090/rest/metadata/datasources GET + +### 4.2 Check if alert stream is creatd +http://localhost:9090/rest/metadata/streams GET + +## 5 Create alert policy and verify alert +### 5.1 create one policy + +http://localhost:9090/rest/metadata/policies POST +``` +{ + "name": "hdfsPolicy", + "description": "hdfsPolicy", + "inputStreams": [ + "hdfs_audit_log_enriched_stream" + ], + "outputStreams": [ + "hdfs_audit_log_enriched_stream_out" + ], + "definition": { + "type": "siddhi", + "value": "from hdfs_audit_log_enriched_stream[user=='hadoop'] select * insert into hdfs_audit_log_enriched_stream_out" + }, + "partitionSpec": [ + { + "streamId": "hdfs_audit_log_enriched_stream", + "type": "GROUPBY", + "columns" : [ + "user" + ] + } + ], + "parallelismHint": 2 +} +``` + +### 5.2 Create alert publishment +``` +{ + "name":"hdfs_audit_log_enriched_stream_out", + "type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher", + "policyIds": [ + "hdfsPolicy" + ], + "properties": { + "subject":"alert when user is hadoop", + "template":"", + "sender": "ea...@apache.org", + "recipients": "ea...@apache.org", + "mail.smtp.host":"", + "connection": "plaintext", + "mail.smtp.port": "25" + }, + "dedupIntervalMin" : "PT1M", + "serializer" : "org.apache.eagle.alert.engine.publisher.impl.StringEventSerializer" +} +``` + +### 5.3 Send message and verify alert +./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667 + +2015-04-24 12:51:31,798 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data dst=null perm=null proto=rpc http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml index 2a8ff0f..2fb7274 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml @@ -155,21 +155,41 @@ <timeseries>true</timeseries> <columns> <column> - <name>action</name> + <name>src</name> <type>string</type> </column> <column> - <name>host</name> + <name>dst</name> <type>string</type> </column> <column> - <name>status</name> + <name>host</name> <type>string</type> </column> <column> <name>timestamp</name> <type>long</type> </column> + <column> + <name>allowed</name> + <type>bool</type> + </column> + <column> + <name>user</name> + <type>string</type> + </column> + <column> + <name>cmd</name> + <type>string</type> + </column> + <column> + <name>sensitivityType</name> + <type>string</type> + </column> + <column> + <name>securityZone</name> + <type>string</type> + </column> </columns> </stream> </streams> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt new file mode 100644 index 0000000..c5f7c9b --- /dev/null +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2 + +./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2 + +./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667 + +./kafka-console-consumer.sh --topic hdfs_audit_log_parsed --zookeeper sandbox.hortonworks.com:2181 --from-beginning http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-server/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index bc9a919..5e07a7d 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -164,6 +164,13 @@ <artifactId>eagle-gc</artifactId> <version>${project.version}</version> </dependency> + + <!-- Alert unit topology as an application --> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-alert-app</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> <resources>
