Repository: incubator-eagle Updated Branches: refs/heads/master 21187b55c -> 9899a58dc
EAGLE-525 refactor oozie to new application framework oozie audit log monitoring to use new app framework https://issues.apache.org/jira/browse/EAGLE-525 Author: @joe-hj <joe.h...@gmail.com> Reviewer: @haoch <h...@apache.org> Closes: #420 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9899a58d Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9899a58d Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9899a58d Branch: refs/heads/master Commit: 9899a58dc3250c677c5df19bc88f02458e72f55a Parents: 21187b5 Author: yonzhang <yonzhang2...@gmail.com> Authored: Wed Sep 7 11:04:20 2016 -0700 Committer: yonzhang <yonzhang2...@gmail.com> Committed: Wed Sep 7 11:04:20 2016 -0700 ---------------------------------------------------------------------- .../ISecurityDataEnrichServiceClient.java | 2 + .../security/service/ISecurityMetadataDAO.java | 2 + .../security/service/InMemMetadataDaoImpl.java | 17 ++++++ .../service/JDBCSecurityMetadataDAO.java | 32 +++++++++++ .../service/OozieSensitivityEntity.java | 50 +++++++++++++++++ .../SecurityDataEnrichServiceClientImpl.java | 17 ++++++ .../SecurityExternalMetadataResource.java | 15 ++++++ .../oozie/parse/OozieAuditLogApplication.java | 4 +- .../oozie/parse/OozieAuditLogParserBolt.java | 32 +++++------ .../OozieResourceSensitivityDataJoinBolt.java | 1 - .../OozieSensitivityDataEnrichLCM.java | 2 +- .../src/main/resources/oozieAuditlogConf.conf | 56 ++++++++++++++++++++ eagle-server/pom.xml | 12 +++++ 13 files changed, 223 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java index 916b08f..4b3f0a1 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java @@ -31,6 +31,8 @@ public interface ISecurityDataEnrichServiceClient extends Closeable, Serializabl OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h); Collection<HdfsSensitivityEntity> listHdfsSensitivities(); OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h); + Collection<OozieSensitivityEntity> listOozieSensitivities(); + OpResult addOozieSensitivity(Collection<OozieSensitivityEntity> h); Collection<IPZoneEntity> listIPZones(); OpResult addIPZone(Collection<IPZoneEntity> h); Collection<HiveSensitivityEntity> listHiveSensitivities(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java index 49d2ed5..a149d1b 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java @@ -27,6 +27,8 @@ public interface ISecurityMetadataDAO { OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h); Collection<HdfsSensitivityEntity> listHdfsSensitivities(); OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h); + Collection<OozieSensitivityEntity> listOozieSensitivities(); + OpResult addOozieSensitivity(Collection<OozieSensitivityEntity> h); Collection<IPZoneEntity> listIPZones(); OpResult addIPZone(Collection<IPZoneEntity> h); Collection<HiveSensitivityEntity> listHiveSensitivities(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java index 2699fe9..359cbe6 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java @@ -38,6 +38,7 @@ public class InMemMetadataDaoImpl implements ISecurityMetadataDAO { private Map<Pair<String, String>, HBaseSensitivityEntity> hBaseSensitivityEntities = new HashMap<>(); private Map<Pair<String, String>, HdfsSensitivityEntity> hdfsSensitivityEntities = new HashMap<>(); + private Map<Pair<String, String>, OozieSensitivityEntity> oozieSensitivityEntities = new HashMap<>(); private Map<String, IPZoneEntity> ipZones = new HashMap<>(); private Map<Pair<String, String>, HiveSensitivityEntity> hiveSensitivityEntities = new HashMap<>(); @@ -73,6 +74,22 @@ public class InMemMetadataDaoImpl implements ISecurityMetadataDAO { return new OpResult(); } + + @Override + public Collection<OozieSensitivityEntity> listOozieSensitivities() { + return oozieSensitivityEntities.values(); + } + + @Override + public OpResult addOozieSensitivity(Collection<OozieSensitivityEntity> h) { + for(OozieSensitivityEntity e : h){ + Pair p = new ImmutablePair<>(e.getJobId(), e.getName()); + oozieSensitivityEntities.put(p, e); + } + return new OpResult(); + } + + @Override public Collection<IPZoneEntity> listIPZones() { return ipZones.values(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java index 05713fc..ecbcfe2 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java @@ -51,6 +51,9 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO { private final String HIVE_QUERY_ALL_STATEMENT = "SELECT site, hive_resource, sensitivity_type FROM hive_sensitivity_entity"; private final String HIVE_INSERT_STATEMENT = "INSERT INTO hive_sensitivity_entity (site, hive_resource, sensitivity_type) VALUES (?, ?, ?)"; + private final String OOZIE_QUERY_ALL_STATEMENT = "SELECT jobid, name, sensitivity_type FROM oozie_sensitivity_entity"; + private final String OOZIE_INSERT_STATEMENT = "INSERT INTO oozie_sensitivity_entity (site, filedir, sensitivity_type) VALUES (?, ?, ?)"; + // get connection url from config public JDBCSecurityMetadataDAO(Config config){ this.config = config; @@ -170,6 +173,35 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO { }); } + + @Override + public Collection<OozieSensitivityEntity> listOozieSensitivities() { + return listEntities(OOZIE_QUERY_ALL_STATEMENT, rs -> { + try { + OozieSensitivityEntity entity = new OozieSensitivityEntity(); + entity.setJobId(rs.getString(1)); + entity.setName(rs.getString(2)); + entity.setSensitivityType(rs.getString(3)); + return entity; + }catch(Exception ex){ throw new IllegalStateException(ex);} + }); + } + + @Override + public OpResult addOozieSensitivity(Collection<OozieSensitivityEntity> h) { + return addEntities(OOZIE_INSERT_STATEMENT, h, (entity, statement) -> { + OozieSensitivityEntity e = (OozieSensitivityEntity)entity; + try { + statement.setString(1, e.getJobId()); + statement.setString(2, e.getName()); + statement.setString(3, e.getSensitivityType()); + }catch(Exception ex){ + throw new IllegalStateException(ex); + } + return statement; + }); + } + @Override public Collection<IPZoneEntity> listIPZones() { return listEntities(IPZONE_QUERY_ALL_STATEMENT, rs -> { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OozieSensitivityEntity.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OozieSensitivityEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OozieSensitivityEntity.java new file mode 100644 index 0000000..464b41d --- /dev/null +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OozieSensitivityEntity.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.security.service; + +/** + * Since 6/10/16. + */ +public class OozieSensitivityEntity { + private String jobId; + private String name; + private String sensitivityType; + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSensitivityType() { + return sensitivityType; + } + + public void setSensitivityType(String sensitivityType) { + this.sensitivityType = sensitivityType; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java index ffabcbc..35e81ac 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java @@ -46,6 +46,9 @@ public class SecurityDataEnrichServiceClientImpl implements ISecurityDataEnrichS private static final String METADATA_LIST_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity"; private static final String METADATA_ADD_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity"; + private static final String METADATA_LIST_OOZIE_SENSITIVITY_PATH = "/metadata/security/oozieSensitivity"; + private static final String METADATA_ADD_OOZIE_SENSITIVITY_PATH = "/metadata/security/oozieSensitivity"; + private static final String METADATA_LIST_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity"; private static final String METADATA_ADD_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity"; @@ -132,6 +135,20 @@ public class SecurityDataEnrichServiceClientImpl implements ISecurityDataEnrichS return new OpResult(); } + + @Override + public Collection<OozieSensitivityEntity> listOozieSensitivities() { + return list(METADATA_LIST_OOZIE_SENSITIVITY_PATH, new GenericType<List<OozieSensitivityEntity>>() { + }); + } + + @Override + public OpResult addOozieSensitivity(Collection<OozieSensitivityEntity> h) { + WebResource r = client.resource(basePath + METADATA_ADD_OOZIE_SENSITIVITY_PATH); + r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h); + return new OpResult(); + } + @Override public Collection<IPZoneEntity> listIPZones(){ return list(METADATA_LIST_IPZONE_PATH, new GenericType<List<IPZoneEntity>>() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityExternalMetadataResource.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityExternalMetadataResource.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityExternalMetadataResource.java index ccf8933..da12117 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityExternalMetadataResource.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityExternalMetadataResource.java @@ -66,6 +66,21 @@ public class SecurityExternalMetadataResource { dao.addHdfsSensitivity(list); } + @Path("/oozieSensitivity") + @GET + @Produces("application/json") + public Collection<OozieSensitivityEntity> getOozieSensitivities(@QueryParam("jobId") String site){ + return dao.listOozieSensitivities(); + } + + @Path("/oozieSensitivity") + @POST + @Consumes("application/json") + public void addOozieSensitivities(Collection<OozieSensitivityEntity> list){ + dao.addOozieSensitivity(list); + } + + @Path("/ipzone") @GET @Produces("application/json") http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogApplication.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogApplication.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogApplication.java index 23fef45..c1c3033 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogApplication.java +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogApplication.java @@ -30,6 +30,7 @@ import backtype.storm.topology.TopologyBuilder; import backtype.storm.tuple.Fields; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import storm.kafka.StringScheme; /** * Since 8/12/16. @@ -54,8 +55,7 @@ public class OozieAuditLogApplication extends StormApplication { OozieAuditLogParserBolt parserBolt = new OozieAuditLogParserBolt(); BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTask); - parserBoltDeclarer.fieldsGrouping("ingest", new Fields("f1")); - + parserBoltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY)); OozieResourceSensitivityDataJoinBolt joinBolt = new OozieResourceSensitivityDataJoinBolt(config); BoltDeclarer boltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParserBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParserBolt.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParserBolt.java index ed59ec1..7f8fa22 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParserBolt.java +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParserBolt.java @@ -57,21 +57,23 @@ public class OozieAuditLogParserBolt extends BaseRichBolt { } catch (Exception ex) { LOG.error("Failing oozie parse audit log message", ex); } - Map<String, Object> map = new TreeMap<String, Object>(); - map.put("timestamp", entity.timestamp); - map.put("level", entity.level); - map.put("ip", entity.ip); - map.put("user", entity.user); - map.put("group", entity.group); - map.put("app", entity.app); - map.put("jobId", entity.jobId); - map.put("operation", entity.operation); - map.put("parameter", entity.parameter); - map.put("status", entity.status); - map.put("httpcode", entity.httpcode); - map.put("errorcode", entity.errorcode); - map.put("errormessage", entity.errormessage); - collector.emit(Arrays.asList(map)); + if (entity != null) { + Map<String, Object> map = new TreeMap<String, Object>(); + map.put("timestamp", entity.timestamp); + map.put("level", entity.level); + map.put("ip", entity.ip); + map.put("user", entity.user); + map.put("group", entity.group); + map.put("app", entity.app); + map.put("jobId", entity.jobId); + map.put("operation", entity.operation); + map.put("parameter", entity.parameter); + map.put("status", entity.status); + map.put("httpcode", entity.httpcode); + map.put("errorcode", entity.errorcode); + map.put("errormessage", entity.errormessage); + collector.emit(Arrays.asList(map)); + } } catch (Exception ex) { LOG.error("error in parsing oozie audit log", ex); } finally { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinBolt.java index 156e326..6e5bc12 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinBolt.java +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinBolt.java @@ -36,7 +36,6 @@ import java.util.regex.Pattern; public class OozieResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<OozieResourceSensitivityAPIEntity, String> { private static final Logger LOG = LoggerFactory.getLogger(OozieResourceSensitivityDataJoinBolt.class); private Config config; - private OutputCollector collector; public OozieResourceSensitivityDataJoinBolt(Config config) { super(config, new OozieSensitivityDataEnrichLCM(config)); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieSensitivityDataEnrichLCM.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieSensitivityDataEnrichLCM.java index eec4058..43d2ee7 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieSensitivityDataEnrichLCM.java +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieSensitivityDataEnrichLCM.java @@ -35,7 +35,7 @@ public class OozieSensitivityDataEnrichLCM extends AbstractDataEnrichLCM { @Override protected Collection loadFromService(ISecurityDataEnrichServiceClient client) { - return null; + return client.listOozieSensitivities(); } @Override http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-security/eagle-security-oozie-auditlog/src/main/resources/oozieAuditlogConf.conf ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/oozieAuditlogConf.conf b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/oozieAuditlogConf.conf new file mode 100755 index 0000000..be5dd51 --- /dev/null +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/oozieAuditlogConf.conf @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "appId" : "OozieAuditLogApp", + "mode" : "LOCAL", + "siteId" : "testsite", + "topology" : { + "numOfTotalWorkers" : 2, + "numOfSpoutTasks" : 1, + "numOfParserTasks" : 2, + "numOfJoinTasks" : 2, + "numOfSensitivityJoinTasks" : 2, + "numOfIPZoneJoinTasks" : 2, + "numOfSinkTasks" : 2 + }, + "dataSourceConfig": { + "topic" : "prd_architect_tandem_oozie_audit_log", + "zkConnection" : "server.eagle.apache.org:2181", + "zkConnectionTimeoutMS" : 15000, + "consumerGroupId" : "EagleConsumer", + "fetchSize" : 1048586, + "deserializerClass" : "org.apache.eagle.security.oozie.parse.OozieAuditLogKafkaDeserializer", + "transactionZKServers" : "server.eagle.apache.org", + "transactionZKPort" : 2181, + "transactionZKRoot" : "/consumers", + "consumerGroupId" : "eagle.ooziesecurity.consumer2", + "transactionStateUpdateMS" : 2000, + "schemeCls" : "storm.kafka.StringScheme" , + "txZkServers" : "server.eagle.apache.org:2181" + }, + "eagleService": { + "host": "localhost", + "port": 9090, + "username": "admin", + "password": "secret" + }, + "dataSinkConfig": { + "topic" : "prd_architect_tandem_oozie_audit_log_parsed2", + "brokerList" : "server.eagle.apache.org:6667", + "serializerClass" : "kafka.serializer.StringEncoder", + "keySerializerClass" : "kafka.serializer.StringEncoder" + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9899a58d/eagle-server/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml index 5e07a7d..16b902a 100644 --- a/eagle-server/pom.xml +++ b/eagle-server/pom.xml @@ -146,6 +146,18 @@ <version>${project.version}</version> </dependency> + <!-- Oozie Auditlog monitoring --> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-security-oozie-auditlog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.eagle</groupId> + <artifactId>eagle-security-oozie-web</artifactId> + <version>${project.version}</version> + </dependency> + <!-- hdfs audit log monitoring --> <dependency> <groupId>org.apache.eagle</groupId>
