METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4 Branch: refs/heads/master Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129 Parents: 9d4842f Author: mmiklavc <michael.miklav...@gmail.com> Authored: Wed Jul 25 08:26:45 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Wed Jul 25 08:26:45 2018 -0600 ---------------------------------------------------------------------- .../java/org/apache/metron/pcap/query/PcapCli.java | 1 - .../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ++++++++++++--- .../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++- .../metron/pcap/finalizer/PcapRestFinalizer.java | 10 ++++++++++ .../metron/pcap/writer/PcapResultsWriter.java | 16 ++++++++++++++++ 5 files changed, 41 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java index 1a23740..0b06b0c 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java @@ -87,7 +87,6 @@ public class PcapCli { try { config = fixedParser.parse(otherArgs); commonConfig = config; - PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" + execDir)); } catch (ParseException | java.text.ParseException e) { System.err.println(e.getMessage()); System.err.flush(); http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java index c379515..c912e58 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java @@ -18,9 +18,13 @@ package org.apache.metron.pcap.finalizer; +import java.io.IOException; +import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.metron.pcap.config.PcapOptions; +import org.apache.metron.pcap.writer.PcapResultsWriter; /** * Write to local FS. @@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer { * as a formatted timestamp + uuid. A final sample format will look as follows: * /base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap */ - private static final String PCAP_CLI_FILENAME_FORMAT = "%s/pcap-data-%s+%04d.pcap"; + private static final String PCAP_CLI_FILENAME_FORMAT = "pcap-data-%s+%04d.pcap"; + + @Override + protected void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig, + List<byte[]> data, Path outputPath) throws IOException { + resultsWriter.writeLocal(data, outputPath.toString()); + } @Override protected Path getOutputPath(Map<String, Object> config, int partition) { - Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config, PcapOptions.STRING_TO_PATH, Path.class); String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config, String.class); - return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath, prefix, partition)); + return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix, partition)); } } http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java index 2c55e15..8dcc401 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java @@ -81,7 +81,7 @@ public abstract class PcapFinalizer implements Finalizer<Path> { for (List<byte[]> data : partitions) { Path outputPath = getOutputPath(config, part++); if (data.size() > 0) { - getResultsWriter().write(hadoopConfig, data, outputPath.toUri().getPath()); + write(resultsWriter, hadoopConfig, data, outputPath); outFiles.add(outputPath); } } @@ -100,6 +100,8 @@ public abstract class PcapFinalizer implements Finalizer<Path> { return new PcapPages(outFiles); } + protected abstract void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig, List<byte[]> data, Path outputPath) throws IOException; + protected abstract Path getOutputPath(Map<String, Object> config, int partition); /** http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java index 95907df..93a3222 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java @@ -18,11 +18,15 @@ package org.apache.metron.pcap.finalizer; +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.metron.job.Statusable; import org.apache.metron.pcap.config.PcapOptions; import java.util.Map; +import org.apache.metron.pcap.writer.PcapResultsWriter; /** * Write to HDFS. @@ -34,6 +38,12 @@ public class PcapRestFinalizer extends PcapFinalizer { private String jobType = Statusable.JobType.MAP_REDUCE.name(); @Override + protected void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig, + List<byte[]> data, Path outputPath) throws IOException { + resultsWriter.write(hadoopConfig, data, outputPath.toString()); + } + + @Override protected Path getOutputPath(Map<String, Object> config, int partition) { String finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config, String.class); String user = PcapOptions.USERNAME.get(config, String.class); http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java index 62ac27c..05d9991 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java @@ -18,6 +18,8 @@ package org.apache.metron.pcap.writer; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -39,6 +41,20 @@ public class PcapResultsWriter { } /** + * Write out pcaps to the local FS + * + * @param pcaps pcap data to write. Pre-merged format as a list of pcaps as byte arrays. + * @param outPath where to write the pcap data to. + * @throws IOException I/O issue encountered. + */ + public void writeLocal(List<byte[]> pcaps, String outPath) throws IOException { + File out = new File(outPath); + try (FileOutputStream fos = new FileOutputStream(out)) { + fos.write(mergePcaps(pcaps)); + } + } + + /** * Creates a pcap file with proper global header from individual pcaps. * * @param pcaps pcap records to merge into a pcap file with header.