[
https://issues.apache.org/jira/browse/NIFI-1975?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15321521#comment-15321521
]
ASF GitHub Bot commented on NIFI-1975:
--------------------------------------
Github user brosander commented on a diff in the pull request:
https://github.com/apache/nifi/pull/492#discussion_r66348434
--- Diff:
nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
---
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.evtx;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.evtx.parser.ChunkHeader;
+import org.apache.nifi.processors.evtx.parser.FileHeader;
+import org.apache.nifi.processors.evtx.parser.FileHeaderFactory;
+import org.apache.nifi.processors.evtx.parser.MalformedChunkException;
+import org.apache.nifi.processors.evtx.parser.Record;
+import org.apache.nifi.processors.evtx.parser.bxml.RootNode;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLStreamException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ParseEvtxTest {
+ public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY =
DocumentBuilderFactory.newInstance();
+ public static final String USER_DATA = "UserData";
+ public static final String EVENT_DATA = "EventData";
+ public static final Set DATA_TAGS = new
HashSet<>(Arrays.asList(EVENT_DATA, USER_DATA));
+
+ @Mock
+ FileHeaderFactory fileHeaderFactory;
+
+ @Mock
+ MalformedChunkHandler malformedChunkHandler;
+
+ @Mock
+ RootNodeHandlerFactory rootNodeHandlerFactory;
+
+ @Mock
+ ResultProcessor resultProcessor;
+
+ @Mock
+ ComponentLog componentLog;
+
+ @Mock
+ InputStream in;
+
+ @Mock
+ OutputStream out;
+
+ @Mock
+ FileHeader fileHeader;
+
+ ParseEvtx parseEvtx;
+
+ @Before
+ public void setup() throws XMLStreamException, IOException {
+ parseEvtx = new ParseEvtx(fileHeaderFactory,
malformedChunkHandler, rootNodeHandlerFactory, resultProcessor);
+ when(fileHeaderFactory.create(in,
componentLog)).thenReturn(fileHeader);
+ }
+
+ @Test
+ public void testGetNameFile() {
+ String basename = "basename";
+ assertEquals(basename + ".xml", parseEvtx.getName(basename, null,
null, ParseEvtx.XML_EXTENSION));
+ }
+
+ @Test
+ public void testGetNameFileChunk() {
+ String basename = "basename";
+ assertEquals(basename + "-chunk1.xml", parseEvtx.getName(basename,
1, null, ParseEvtx.XML_EXTENSION));
+ }
+
+ @Test
+ public void testGetNameFileChunkRecord() {
+ String basename = "basename";
+ assertEquals(basename + "-chunk1-record2.xml",
parseEvtx.getName(basename, 1, 2, ParseEvtx.XML_EXTENSION));
+ }
+
+ @Test
+ public void testGetBasenameEvtxExtension() {
+ String basename = "basename";
+ FlowFile flowFile = mock(FlowFile.class);
+
+
when(flowFile.getAttribute(CoreAttributes.FILENAME.key())).thenReturn(basename
+ ".evtx");
+
+ assertEquals(basename, parseEvtx.getBasename(flowFile,
componentLog));
+ verifyNoMoreInteractions(componentLog);
+ }
+
+ @Test
+ public void testGetBasenameExtension() {
+ String basename = "basename.wrongextension";
+ FlowFile flowFile = mock(FlowFile.class);
+ ComponentLog componentLog = mock(ComponentLog.class);
+
+
when(flowFile.getAttribute(CoreAttributes.FILENAME.key())).thenReturn(basename);
+
+ assertEquals(basename, parseEvtx.getBasename(flowFile,
componentLog));
+ verify(componentLog).warn(anyString(), isA(Object[].class));
+ }
+
+ @Test
+ public void testGetRelationships() {
+ assertEquals(ParseEvtx.RELATIONSHIPS,
parseEvtx.getRelationships());
+ }
+
+ @Test
+ public void testGetSupportedPropertyDescriptors() {
+ assertEquals(ParseEvtx.PROPERTY_DESCRIPTORS,
parseEvtx.getSupportedPropertyDescriptors());
+ }
+
+ @Test
+ public void testProcessFileGranularity() throws IOException,
MalformedChunkException, XMLStreamException {
+ String basename = "basename";
+ int chunkNum = 5;
+ int offset = 10001;
+ byte[] badChunk = {8};
+
+ RootNodeHandler rootNodeHandler = mock(RootNodeHandler.class);
+
when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler);
+ ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
+ ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
+ Record record1 = mock(Record.class);
+ Record record2 = mock(Record.class);
+ Record record3 = mock(Record.class);
+ RootNode rootNode1 = mock(RootNode.class);
+ RootNode rootNode2 = mock(RootNode.class);
+ RootNode rootNode3 = mock(RootNode.class);
+ ProcessSession session = mock(ProcessSession.class);
+ FlowFile flowFile = mock(FlowFile.class);
+ AtomicReference<Exception> reference = new AtomicReference<>();
+ MalformedChunkException malformedChunkException = new
MalformedChunkException("Test", null, offset, chunkNum, badChunk);
+
+ when(record1.getRootNode()).thenReturn(rootNode1);
+ when(record2.getRootNode()).thenReturn(rootNode2);
+ when(record3.getRootNode()).thenReturn(rootNode3);
+
+
when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
+
+ when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
+ when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
+
+
when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
+
+ parseEvtx.processFileGranularity(session, componentLog, flowFile,
basename, reference, in, out);
+
+ verify(malformedChunkHandler).handle(flowFile, session,
parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION),
badChunk);
+ verify(rootNodeHandler).handle(rootNode1);
+ verify(rootNodeHandler).handle(rootNode2);
+ verify(rootNodeHandler).handle(rootNode3);
+ verify(rootNodeHandler).close();
+ }
+
+ @Test
+ public void testProcessChunkGranularity() throws IOException,
MalformedChunkException, XMLStreamException {
+ String basename = "basename";
+ int chunkNum = 5;
+ int offset = 10001;
+ byte[] badChunk = {8};
+
+ RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class);
+ RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class);
+ OutputStream out2 = mock(OutputStream.class);
+
when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1);
+
when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2);
+ ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
+ ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
+ Record record1 = mock(Record.class);
+ Record record2 = mock(Record.class);
+ Record record3 = mock(Record.class);
+ RootNode rootNode1 = mock(RootNode.class);
+ RootNode rootNode2 = mock(RootNode.class);
+ RootNode rootNode3 = mock(RootNode.class);
+ ProcessSession session = mock(ProcessSession.class);
+ FlowFile flowFile = mock(FlowFile.class);
+ FlowFile created1 = mock(FlowFile.class);
+ FlowFile updated1 = mock(FlowFile.class);
+ FlowFile created2 = mock(FlowFile.class);
+ FlowFile updated2 = mock(FlowFile.class);
+ MalformedChunkException malformedChunkException = new
MalformedChunkException("Test", null, offset, chunkNum, badChunk);
+
+
when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(null);
+
+ when(session.write(eq(created1),
any(OutputStreamCallback.class))).thenAnswer(invocation -> {
+ ((OutputStreamCallback)
invocation.getArguments()[1]).process(out);
+ return updated1;
+ });
+
+ when(session.write(eq(created2),
any(OutputStreamCallback.class))).thenAnswer(invocation -> {
+ ((OutputStreamCallback)
invocation.getArguments()[1]).process(out2);
+ return updated2;
+ });
+
+ when(record1.getRootNode()).thenReturn(rootNode1);
+ when(record2.getRootNode()).thenReturn(rootNode2);
+ when(record3.getRootNode()).thenReturn(rootNode3);
+
+
when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
+
+ when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
+ when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
+
+
when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
+
+ parseEvtx.processChunkGranularity(session, componentLog, flowFile,
basename, in);
+
+ verify(malformedChunkHandler).handle(flowFile, session,
parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION),
badChunk);
+ verify(rootNodeHandler1).handle(rootNode1);
+ verify(rootNodeHandler1).close();
+ verify(rootNodeHandler2).handle(rootNode2);
+ verify(rootNodeHandler2).handle(rootNode3);
+ verify(rootNodeHandler2).close();
+ }
+
+ @Test
+ public void testProcess1RecordGranularity() throws IOException,
MalformedChunkException, XMLStreamException {
+ String basename = "basename";
+ int chunkNum = 5;
+ int offset = 10001;
+ byte[] badChunk = {8};
+
+ RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class);
+ RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class);
+ RootNodeHandler rootNodeHandler3 = mock(RootNodeHandler.class);
+ OutputStream out2 = mock(OutputStream.class);
+ OutputStream out3 = mock(OutputStream.class);
+
when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1);
+
when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2);
+
when(rootNodeHandlerFactory.create(out3)).thenReturn(rootNodeHandler3);
+ ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
+ ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
+ Record record1 = mock(Record.class);
+ Record record2 = mock(Record.class);
+ Record record3 = mock(Record.class);
+ RootNode rootNode1 = mock(RootNode.class);
+ RootNode rootNode2 = mock(RootNode.class);
+ RootNode rootNode3 = mock(RootNode.class);
+ ProcessSession session = mock(ProcessSession.class);
+ FlowFile flowFile = mock(FlowFile.class);
+ FlowFile created1 = mock(FlowFile.class);
+ FlowFile updated1 = mock(FlowFile.class);
+ FlowFile created2 = mock(FlowFile.class);
+ FlowFile updated2 = mock(FlowFile.class);
+ FlowFile created3 = mock(FlowFile.class);
+ FlowFile updated3 = mock(FlowFile.class);
+ MalformedChunkException malformedChunkException = new
MalformedChunkException("Test", null, offset, chunkNum, badChunk);
+
+
when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(created3).thenReturn(null);
+
+ when(session.write(eq(created1),
any(OutputStreamCallback.class))).thenAnswer(invocation -> {
+ ((OutputStreamCallback)
invocation.getArguments()[1]).process(out);
+ return updated1;
+ });
+
+ when(session.write(eq(created2),
any(OutputStreamCallback.class))).thenAnswer(invocation -> {
+ ((OutputStreamCallback)
invocation.getArguments()[1]).process(out2);
+ return updated2;
+ });
+
+ when(session.write(eq(created3),
any(OutputStreamCallback.class))).thenAnswer(invocation -> {
+ ((OutputStreamCallback)
invocation.getArguments()[1]).process(out3);
+ return updated3;
+ });
+
+ when(record1.getRootNode()).thenReturn(rootNode1);
+ when(record2.getRootNode()).thenReturn(rootNode2);
+ when(record3.getRootNode()).thenReturn(rootNode3);
+
+
when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
+
+ when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
+ when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
+
+
when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
+
+ parseEvtx.processRecordGranularity(session, componentLog,
flowFile, basename, in);
+
+ verify(malformedChunkHandler).handle(flowFile, session,
parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION),
badChunk);
+ verify(rootNodeHandler1).handle(rootNode1);
+ verify(rootNodeHandler1).close();
+ verify(rootNodeHandler2).handle(rootNode2);
+ verify(rootNodeHandler2).close();
+ verify(rootNodeHandler3).handle(rootNode3);
+ verify(rootNodeHandler3).close();
+ }
+
+ @Test
+ public void fileGranularityLifecycleTest() throws IOException,
ParserConfigurationException, SAXException {
--- End diff --
@mattyb149 lifecycle tests of the 3 different granularities using a
doctored evtx file that has good chunks, bad chunk headers, and a malformed
record
> Processor to Parse .evtx files
> ------------------------------
>
> Key: NIFI-1975
> URL: https://issues.apache.org/jira/browse/NIFI-1975
> Project: Apache NiFi
> Issue Type: Sub-task
> Reporter: Bryan Rosander
>
> Windows event logs are stored in .evtx format as-of Windows Vista. If we
> port the pure python implementation of an evtx parser at
> https://github.com/williballenthin/python-evtx to Java, we should be able to
> ingest those files in NiFi on any operating system
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
