[09/13] metron git commit: METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via mmiklavc) closes apache/metron#1011

2018-05-18 Thread leet 
METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via 
mmiklavc) closes apache/metron#1011


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/20eaed23
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/20eaed23
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/20eaed23

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 20eaed239b2552d0823d34f571b63d941c352bc9
Parents: 9ce4ba5
Author: mmiklavc 
Authored: Tue May 15 11:12:07 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue May 15 11:12:07 2018 -0600

--
 .../common/utils/CompressionStrategies.java | 100 +++
 .../common/utils/CompressionStrategy.java   |  52 ++
 .../common/utils/CompressionUtilsTest.java  |  62 
 .../nonbulk/geo/GeoEnrichmentLoader.java|  70 +
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java|  40 ++--
 5 files changed, 298 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/20eaed23/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
new file mode 100644
index 000..f9c53c8
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipException;
+
+/*
+ * Factory to provide various compression strategies.
+ */
+public enum CompressionStrategies implements CompressionStrategy {
+
+  GZIP(new CompressionStrategy() {
+@Override
+public void compress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  FileOutputStream fos = new FileOutputStream(outFile);
+  GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = fis.read(buffer)) != -1) {
+  gzipOS.write(buffer, 0, len);
+}
+  }
+}
+
+@Override
+public void decompress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  GZIPInputStream gis = new GZIPInputStream(fis);
+  FileOutputStream fos = new FileOutputStream(outFile)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = gis.read(buffer)) != -1) {
+  fos.write(buffer, 0, len);
+}
+  }
+
+}
+
+@Override
+public boolean test(File gzipFile) {
+  try (FileInputStream fis = new FileInputStream(gzipFile);
+  GZIPInputStream gis = new GZIPInputStream(fis)) {
+byte[] buffer = new byte[1024];
+// this will throw an exception on malformed file
+gis.read(buffer);
+  } catch (ZipException | EOFException e) {
+return false;
+  } catch (IOException e) {
+throw new IllegalStateException("Error occurred while attempting to 
validate gzip file", e);
+  }
+  return true;
+}
+  });
+
+  private CompressionStrategy strategy;
+
+  CompressionStrategies(CompressionStrategy strategy) {
+this.strategy = strategy;
+  }
+
+  @Override
+  public void compress(File inFile, File outFile) throws IOException {
+strategy.compress(inFile, outFile);
+  }
+
+  @Override
+  public void decompress(File inFile, File outFile) throws

metron git commit: METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via mmiklavc) closes apache/metron#1011

2018-05-15 Thread mmiklavcic 
Repository: metron
Updated Branches:
  refs/heads/master 9ce4ba5a9 -> 20eaed239


METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via 
mmiklavc) closes apache/metron#1011


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/20eaed23
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/20eaed23
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/20eaed23

Branch: refs/heads/master
Commit: 20eaed239b2552d0823d34f571b63d941c352bc9
Parents: 9ce4ba5
Author: mmiklavc 
Authored: Tue May 15 11:12:07 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue May 15 11:12:07 2018 -0600

--
 .../common/utils/CompressionStrategies.java | 100 +++
 .../common/utils/CompressionStrategy.java   |  52 ++
 .../common/utils/CompressionUtilsTest.java  |  62 
 .../nonbulk/geo/GeoEnrichmentLoader.java|  70 +
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java|  40 ++--
 5 files changed, 298 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/20eaed23/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
new file mode 100644
index 000..f9c53c8
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipException;
+
+/*
+ * Factory to provide various compression strategies.
+ */
+public enum CompressionStrategies implements CompressionStrategy {
+
+  GZIP(new CompressionStrategy() {
+@Override
+public void compress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  FileOutputStream fos = new FileOutputStream(outFile);
+  GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = fis.read(buffer)) != -1) {
+  gzipOS.write(buffer, 0, len);
+}
+  }
+}
+
+@Override
+public void decompress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  GZIPInputStream gis = new GZIPInputStream(fis);
+  FileOutputStream fos = new FileOutputStream(outFile)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = gis.read(buffer)) != -1) {
+  fos.write(buffer, 0, len);
+}
+  }
+
+}
+
+@Override
+public boolean test(File gzipFile) {
+  try (FileInputStream fis = new FileInputStream(gzipFile);
+  GZIPInputStream gis = new GZIPInputStream(fis)) {
+byte[] buffer = new byte[1024];
+// this will throw an exception on malformed file
+gis.read(buffer);
+  } catch (ZipException | EOFException e) {
+return false;
+  } catch (IOException e) {
+throw new IllegalStateException("Error occurred while attempting to 
validate gzip file", e);
+  }
+  return true;
+}
+  });
+
+  private CompressionStrategy strategy;
+
+  CompressionStrategies(CompressionStrategy strategy) {
+this.strategy = strategy;
+  }
+
+  @Override
+  public void compress(File inFile, File outFile) throws IOException {
+strategy.compress(inFile, outFile);
+  }
+
+  @Override
+