metron git commit: METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141

2018-08-15 Thread mmiklavcic 
Repository: metron
Updated Branches:
  refs/heads/master 2b0060b06 -> 308c2b27b


METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack 
for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/308c2b27
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/308c2b27
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/308c2b27

Branch: refs/heads/master
Commit: 308c2b27b9b234c0c84c343dfc45b3617b027e64
Parents: 2b0060b
Author: MohanDV 
Authored: Wed Aug 15 16:16:26 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Aug 15 16:20:54 2018 -0600

--
 metron-deployment/Kerberos-manual-setup.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/308c2b27/metron-deployment/Kerberos-manual-setup.md
--
diff --git a/metron-deployment/Kerberos-manual-setup.md 
b/metron-deployment/Kerberos-manual-setup.md
index e3de148..18891a4 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -580,7 +580,7 @@ X-Pack
 Submit the update to Zookeeper
 
 ```
-$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i 
METRON_HOME/config/zookeeper/ -z $ZOOKEEPER
+$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i 
$METRON_HOME/config/zookeeper/ -z $ZOOKEEPER
 ```
 
 1. The last step before restarting the topology is to create a custom X-Pack 
shaded and relocated jar. This is up to you because of licensing restrictions, 
but here is a sample Maven pom file that should help.
@@ -737,6 +737,8 @@ X-Pack
 $METRON_HOME/bin/start_elasticsearch_topology.sh
 ```
 
+1. Restart the metron-rest service, and make sure the 
elasticsearch-xpack-shaded-5.6.2.jar is in the METRON_REST_CLASSPATH when the 
metron-rest starts.
+
 Once you've performed these steps, you should be able to start seeing data in 
your ES indexes.
 
 ### X-Pack Common Problems

metron git commit: METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166

2018-08-15 Thread mmiklavcic 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel 6b70571d6 -> 076a6a197


METRON-1738: Pcap directories should have correct permissions (merrimanr via 
mmiklavc) closes apache/metron#1166


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/076a6a19
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/076a6a19
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/076a6a19

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 076a6a197df629d3b04e0757320e7681ea2fb3d9
Parents: 6b70571
Author: merrimanr 
Authored: Wed Aug 15 16:02:47 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Aug 15 16:02:47 2018 -0600

--
 .../package/scripts/params/params_linux.py  |  1 +
 .../package/scripts/params/status_params.py |  1 +
 .../CURRENT/package/scripts/rest_commands.py| 23 +++-
 .../CURRENT/package/scripts/rest_master.py  |  5 +
 4 files changed, 24 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 115a54c..9be09f1 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -393,6 +393,7 @@ pcap_page_size = 
config['configurations']['metron-rest-env']['pcap_page_size']
 pcap_yarn_queue = 
config['configurations']['metron-rest-env']['pcap_yarn_queue']
 pcap_finalizer_threadpool_size= 
config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
 pcap_configured_flag_file = status_params.pcap_configured_flag_file
+pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
 
 # MapReduce
 metron_user_hdfs_dir = '/user/' + metron_user

http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 0a9fdd0..99f5ec0 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -120,6 +120,7 @@ metron_keytab_path = 
config['configurations']['metron-env']['metron_service_keyt
 
 # Pcap
 pcap_configured_flag_file = metron_zookeeper_config_path + 
'/../metron_pcap_configured'
+pcap_perm_configured_flag_file = metron_zookeeper_config_path + 
'/../metron_pcap_perm_configured'
 
 # MapReduce
 metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + 
'/../metron_user_hdfs_dir_configured'
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 463dca1..d44f478 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -38,6 +38,7 @@ class RestCommands:
 __hbase_configured = False
 __hbase_acl_configured = False
 __pcap_configured = False
+__pcap_perm_configured = False
 __metron_user_hdfs_dir_configured = False
 
 def __in

metron git commit: METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes apache/metron#1164

2018-08-15 Thread mmiklavcic 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel d9e1f381c -> 6b70571d6


METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes 
apache/metron#1164


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6b70571d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6b70571d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6b70571d

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 6b70571d6de3951c98269bbf5b38e8b69deddfab
Parents: d9e1f38
Author: merrimanr 
Authored: Wed Aug 15 16:00:13 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Aug 15 16:00:13 2018 -0600

--
 metron-interface/metron-rest/README.md | 11 +++
 1 file changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/6b70571d/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md 
b/metron-interface/metron-rest/README.md
index 080422d..2c216d1 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -222,6 +222,17 @@ Out of the box it is a simple wrapper around the tshark 
command to transform raw
 REST will supply the script with raw pcap data through standard in and expects 
PDML data serialized as XML.
 
 Pcap query jobs can be configured for submission to a YARN queue.  This 
setting is exposed as the Spring property `pcap.yarn.queue`.  If configured, 
the REST application will set the `mapreduce.job.queuename` Hadoop property to 
that value.
+It is highly recommended that a dedicated YARN queue be created and configured 
for Pcap queries to prevent a job from consuming too many cluster resources.  
More information about setting up YARN queues can be found 
[here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
+
+Pcap query results are stored in HDFS.  The location of query results when run 
through the REST app is determined by a couple factors.  The root of Pcap query 
results defaults to `/apps/metron/pcap/output` but can be changed with the 
+Spring property `pcap.final.output.path`.  Assuming the default Pcap query 
output directory, the path to a result page will follow this pattern:
+```
+/apps/metron/pcap/output/{username}/MAP_REDUCE/{job id}/page-{page number}.pcap
+```
+Over time Pcap query results will accumulate in HDFS.  Currently these results 
are not cleaned up automatically so cluster administrators should be aware of 
this and monitor them.  It is highly recommended that a process be put in place 
to 
+periodically delete files and directories under the Pcap query results root.
+
+Users should also be mindful of date ranges used in queries so they don't 
produce result sets that are too large.  Currently there are no limits enforced 
on date ranges.
 
 Queries can also be configured on a global level for setting the number of 
results per page via a Spring property `pcap.page.size`. By default, this value 
is set to 10 pcaps per page, but you may choose to set this value higher
 based on observing frequenetly-run query result sizes. This setting works in 
conjunction with the property for setting finalizer threadpool size when 
optimizing query performance.

metron git commit: METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157

2018-08-15 Thread mmiklavcic 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel 8a926dd5a -> d9e1f381c


METRON-1732: Fix job status liveness bug and parallelize finalizer file writing 
(mmiklavc via mmiklavc) closes apache/metron#1157


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d9e1f381
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d9e1f381
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d9e1f381

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: d9e1f381cc1ae9edfbffc7216abffdbd8f942c09
Parents: 8a926dd
Author: mmiklavc 
Authored: Wed Aug 15 11:30:10 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Aug 15 11:30:10 2018 -0600

--
 .../CURRENT/configuration/metron-rest-env.xml   | 14 +++
 .../package/scripts/params/params_linux.py  |  1 +
 .../METRON/CURRENT/package/templates/metron.j2  |  1 +
 .../METRON/CURRENT/themes/metron_theme.json | 10 +++
 metron-interface/metron-rest/README.md  | 10 +++
 .../src/main/config/rest_application.yml|  1 +
 .../apache/metron/rest/MetronRestConstants.java |  1 +
 .../rest/service/impl/PcapServiceImpl.java  | 25 +++---
 .../apache/metron/rest/mock/MockPcapJob.java| 18 ++--
 .../rest/service/impl/PcapServiceImplTest.java  | 49 ++-
 .../org/apache/metron/pcap/query/CliParser.java | 10 ++-
 .../PcapTopologyIntegrationTest.java|  1 +
 .../apache/metron/pcap/query/PcapCliTest.java   | 21 ++---
 .../java/org/apache/metron/pcap/PcapPages.java  |  2 +-
 .../apache/metron/pcap/config/PcapConfig.java   |  6 ++
 .../metron/pcap/config/PcapGlobalDefaults.java  |  1 +
 .../apache/metron/pcap/config/PcapOptions.java  |  3 +-
 .../metron/pcap/finalizer/PcapFinalizer.java| 70 +--
 .../java/org/apache/metron/pcap/mr/PcapJob.java | 90 ++--
 .../org/apache/metron/pcap/mr/PcapJobTest.java  | 16 +++-
 20 files changed, 258 insertions(+), 92 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 895c091..767afa3 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -201,5 +201,19 @@
 true
 
 
+
+pcap_finalizer_threadpool_size
+Pcap Finalizer Threadpool Size
+The number of threads to use when finalizing Pcap jobs. 
This affects parallelism
+  around writing out paged files to their final location.
+  If it's a string and ends with "C", then strip the C and treat it as 
an integral multiple of
+  the number of cores. If it's a string and does not end with a C, 
then treat it as a number in
+  string form.
+
+1
+
+false
+
+
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 4f8a9a7..115a54c 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -391,6 +391,7 @@ pcap_base_interim_result_path = 
config['configurations']['metron-rest-env']['pca
 pcap_final_output_path = 
config['configurations']['metron-rest-env']['pcap_final_output_path']
 pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
 pcap_yarn_queue = 
config['configurations']['metron-rest-env']['pcap_yarn_queue']
+pcap_finalizer_threadpool_size= 
config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
 pcap_configured_flag_file = status_para

metron git commit: METRON-1703 Make Core Profiler Components Serializable (nickwallen) closes apache/metron#1145

2018-08-15 Thread nickallen 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1699-create-batch-profiler a1e25a4bf -> 4fb920167


METRON-1703 Make Core Profiler Components Serializable (nickwallen) closes 
apache/metron#1145


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4fb92016
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4fb92016
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4fb92016

Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 4fb9201670acf9960087f004f8858e99c0a87981
Parents: a1e25a4
Author: nickwallen 
Authored: Wed Aug 15 09:26:42 2018 -0400
Committer: nickallen 
Committed: Wed Aug 15 09:26:42 2018 -0400

--
 .../profiler/DefaultMessageDistributor.java | 28 +++-
 .../metron/profiler/DefaultMessageRouter.java   |  3 ++-
 .../apache/metron/profiler/MessageRoute.java|  4 ++-
 .../profiler/clock/DefaultClockFactory.java |  4 ++-
 .../metron/profiler/clock/EventTimeClock.java   |  3 ++-
 5 files changed, 25 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/4fb92016/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
index 82f7174..c926a70 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
@@ -20,19 +20,11 @@
 
 package org.apache.metron.profiler;
 
-import static java.lang.String.format;
-
 import com.google.common.base.Ticker;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 import org.apache.metron.stellar.dsl.Context;
@@ -40,6 +32,16 @@ import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.String.format;
+
 /**
  * The default implementation of a {@link MessageDistributor}.
  *
@@ -57,7 +59,7 @@ import org.slf4j.LoggerFactory;
  * lost.
  *
  */
-public class DefaultMessageDistributor implements MessageDistributor {
+public class DefaultMessageDistributor implements MessageDistributor, 
Serializable {
 
   protected static final Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
@@ -73,7 +75,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
* messages.  Once it has not received messages for a period of time, it is
* moved to the expired cache.
*/
-  private transient Cache activeCache;
+  private Cache activeCache;
 
   /**
* A cache of expired profiles.
@@ -84,7 +86,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
* can flush the state of the expired profile.  If the client does not flush
* the expired profiles, this state will be lost forever.
*/
-  private transient Cache expiredCache;
+  private Cache expiredCache;
 
   /**
* Create a new message distributor.
@@ -287,7 +289,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
   /**
* A listener that is notified when profiles expire from the active cache.
*/
-  private class ActiveCacheRemovalListener implements RemovalListener {
+  private class ActiveCacheRemovalListener implements RemovalListener, Serializable {
 
 @Override
 public void onRemoval(RemovalNotification 
notification) {
@@ -305,7 +307,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
   /**
* A listener that is notified when profiles expire from the active cache.
*/
-  private class ExpiredCacheRemovalListener implements 
RemovalListener {
+  private class ExpiredCacheRemovalListener implements 
RemovalListener, Serializable {
 
 @Override
 public void onRe