[ https://issues.apache.org/jira/browse/NIFI-1975?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15321521#comment-15321521 ]
ASF GitHub Bot commented on NIFI-1975: -------------------------------------- Github user brosander commented on a diff in the pull request: https://github.com/apache/nifi/pull/492#discussion_r66348434 --- Diff: nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java --- @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.evtx; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.io.OutputStreamCallback; +import org.apache.nifi.processors.evtx.parser.ChunkHeader; +import org.apache.nifi.processors.evtx.parser.FileHeader; +import org.apache.nifi.processors.evtx.parser.FileHeaderFactory; +import org.apache.nifi.processors.evtx.parser.MalformedChunkException; +import org.apache.nifi.processors.evtx.parser.Record; +import org.apache.nifi.processors.evtx.parser.bxml.RootNode; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ParseEvtxTest { + public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); + public static final String USER_DATA = "UserData"; + public static final String EVENT_DATA = "EventData"; + public static final Set DATA_TAGS = new HashSet<>(Arrays.asList(EVENT_DATA, USER_DATA)); + + @Mock + FileHeaderFactory fileHeaderFactory; + + @Mock + MalformedChunkHandler malformedChunkHandler; + + @Mock + RootNodeHandlerFactory rootNodeHandlerFactory; + + @Mock + ResultProcessor resultProcessor; + + @Mock + ComponentLog componentLog; + + @Mock + InputStream in; + + @Mock + OutputStream out; + + @Mock + FileHeader fileHeader; + + ParseEvtx parseEvtx; + + @Before + public void setup() throws XMLStreamException, IOException { + parseEvtx = new ParseEvtx(fileHeaderFactory, malformedChunkHandler, rootNodeHandlerFactory, resultProcessor); + when(fileHeaderFactory.create(in, componentLog)).thenReturn(fileHeader); + } + + @Test + public void testGetNameFile() { + String basename = "basename"; + assertEquals(basename + ".xml", parseEvtx.getName(basename, null, null, ParseEvtx.XML_EXTENSION)); + } + + @Test + public void testGetNameFileChunk() { + String basename = "basename"; + assertEquals(basename + "-chunk1.xml", parseEvtx.getName(basename, 1, null, ParseEvtx.XML_EXTENSION)); + } + + @Test + public void testGetNameFileChunkRecord() { + String basename = "basename"; + assertEquals(basename + "-chunk1-record2.xml", parseEvtx.getName(basename, 1, 2, ParseEvtx.XML_EXTENSION)); + } + + @Test + public void testGetBasenameEvtxExtension() { + String basename = "basename"; + FlowFile flowFile = mock(FlowFile.class); + + when(flowFile.getAttribute(CoreAttributes.FILENAME.key())).thenReturn(basename + ".evtx"); + + assertEquals(basename, parseEvtx.getBasename(flowFile, componentLog)); + verifyNoMoreInteractions(componentLog); + } + + @Test + public void testGetBasenameExtension() { + String basename = "basename.wrongextension"; + FlowFile flowFile = mock(FlowFile.class); + ComponentLog componentLog = mock(ComponentLog.class); + + when(flowFile.getAttribute(CoreAttributes.FILENAME.key())).thenReturn(basename); + + assertEquals(basename, parseEvtx.getBasename(flowFile, componentLog)); + verify(componentLog).warn(anyString(), isA(Object[].class)); + } + + @Test + public void testGetRelationships() { + assertEquals(ParseEvtx.RELATIONSHIPS, parseEvtx.getRelationships()); + } + + @Test + public void testGetSupportedPropertyDescriptors() { + assertEquals(ParseEvtx.PROPERTY_DESCRIPTORS, parseEvtx.getSupportedPropertyDescriptors()); + } + + @Test + public void testProcessFileGranularity() throws IOException, MalformedChunkException, XMLStreamException { + String basename = "basename"; + int chunkNum = 5; + int offset = 10001; + byte[] badChunk = {8}; + + RootNodeHandler rootNodeHandler = mock(RootNodeHandler.class); + when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler); + ChunkHeader chunkHeader1 = mock(ChunkHeader.class); + ChunkHeader chunkHeader2 = mock(ChunkHeader.class); + Record record1 = mock(Record.class); + Record record2 = mock(Record.class); + Record record3 = mock(Record.class); + RootNode rootNode1 = mock(RootNode.class); + RootNode rootNode2 = mock(RootNode.class); + RootNode rootNode3 = mock(RootNode.class); + ProcessSession session = mock(ProcessSession.class); + FlowFile flowFile = mock(FlowFile.class); + AtomicReference<Exception> reference = new AtomicReference<>(); + MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk); + + when(record1.getRootNode()).thenReturn(rootNode1); + when(record2.getRootNode()).thenReturn(rootNode2); + when(record3.getRootNode()).thenReturn(rootNode3); + + when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false); + when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null); + + when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false); + when(chunkHeader1.next()).thenReturn(record1).thenReturn(null); + + when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false); + when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null); + + parseEvtx.processFileGranularity(session, componentLog, flowFile, basename, reference, in, out); + + verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk); + verify(rootNodeHandler).handle(rootNode1); + verify(rootNodeHandler).handle(rootNode2); + verify(rootNodeHandler).handle(rootNode3); + verify(rootNodeHandler).close(); + } + + @Test + public void testProcessChunkGranularity() throws IOException, MalformedChunkException, XMLStreamException { + String basename = "basename"; + int chunkNum = 5; + int offset = 10001; + byte[] badChunk = {8}; + + RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class); + RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class); + OutputStream out2 = mock(OutputStream.class); + when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1); + when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2); + ChunkHeader chunkHeader1 = mock(ChunkHeader.class); + ChunkHeader chunkHeader2 = mock(ChunkHeader.class); + Record record1 = mock(Record.class); + Record record2 = mock(Record.class); + Record record3 = mock(Record.class); + RootNode rootNode1 = mock(RootNode.class); + RootNode rootNode2 = mock(RootNode.class); + RootNode rootNode3 = mock(RootNode.class); + ProcessSession session = mock(ProcessSession.class); + FlowFile flowFile = mock(FlowFile.class); + FlowFile created1 = mock(FlowFile.class); + FlowFile updated1 = mock(FlowFile.class); + FlowFile created2 = mock(FlowFile.class); + FlowFile updated2 = mock(FlowFile.class); + MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk); + + when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(null); + + when(session.write(eq(created1), any(OutputStreamCallback.class))).thenAnswer(invocation -> { + ((OutputStreamCallback) invocation.getArguments()[1]).process(out); + return updated1; + }); + + when(session.write(eq(created2), any(OutputStreamCallback.class))).thenAnswer(invocation -> { + ((OutputStreamCallback) invocation.getArguments()[1]).process(out2); + return updated2; + }); + + when(record1.getRootNode()).thenReturn(rootNode1); + when(record2.getRootNode()).thenReturn(rootNode2); + when(record3.getRootNode()).thenReturn(rootNode3); + + when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false); + when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null); + + when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false); + when(chunkHeader1.next()).thenReturn(record1).thenReturn(null); + + when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false); + when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null); + + parseEvtx.processChunkGranularity(session, componentLog, flowFile, basename, in); + + verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk); + verify(rootNodeHandler1).handle(rootNode1); + verify(rootNodeHandler1).close(); + verify(rootNodeHandler2).handle(rootNode2); + verify(rootNodeHandler2).handle(rootNode3); + verify(rootNodeHandler2).close(); + } + + @Test + public void testProcess1RecordGranularity() throws IOException, MalformedChunkException, XMLStreamException { + String basename = "basename"; + int chunkNum = 5; + int offset = 10001; + byte[] badChunk = {8}; + + RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class); + RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class); + RootNodeHandler rootNodeHandler3 = mock(RootNodeHandler.class); + OutputStream out2 = mock(OutputStream.class); + OutputStream out3 = mock(OutputStream.class); + when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1); + when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2); + when(rootNodeHandlerFactory.create(out3)).thenReturn(rootNodeHandler3); + ChunkHeader chunkHeader1 = mock(ChunkHeader.class); + ChunkHeader chunkHeader2 = mock(ChunkHeader.class); + Record record1 = mock(Record.class); + Record record2 = mock(Record.class); + Record record3 = mock(Record.class); + RootNode rootNode1 = mock(RootNode.class); + RootNode rootNode2 = mock(RootNode.class); + RootNode rootNode3 = mock(RootNode.class); + ProcessSession session = mock(ProcessSession.class); + FlowFile flowFile = mock(FlowFile.class); + FlowFile created1 = mock(FlowFile.class); + FlowFile updated1 = mock(FlowFile.class); + FlowFile created2 = mock(FlowFile.class); + FlowFile updated2 = mock(FlowFile.class); + FlowFile created3 = mock(FlowFile.class); + FlowFile updated3 = mock(FlowFile.class); + MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk); + + when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(created3).thenReturn(null); + + when(session.write(eq(created1), any(OutputStreamCallback.class))).thenAnswer(invocation -> { + ((OutputStreamCallback) invocation.getArguments()[1]).process(out); + return updated1; + }); + + when(session.write(eq(created2), any(OutputStreamCallback.class))).thenAnswer(invocation -> { + ((OutputStreamCallback) invocation.getArguments()[1]).process(out2); + return updated2; + }); + + when(session.write(eq(created3), any(OutputStreamCallback.class))).thenAnswer(invocation -> { + ((OutputStreamCallback) invocation.getArguments()[1]).process(out3); + return updated3; + }); + + when(record1.getRootNode()).thenReturn(rootNode1); + when(record2.getRootNode()).thenReturn(rootNode2); + when(record3.getRootNode()).thenReturn(rootNode3); + + when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false); + when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null); + + when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false); + when(chunkHeader1.next()).thenReturn(record1).thenReturn(null); + + when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false); + when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null); + + parseEvtx.processRecordGranularity(session, componentLog, flowFile, basename, in); + + verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk); + verify(rootNodeHandler1).handle(rootNode1); + verify(rootNodeHandler1).close(); + verify(rootNodeHandler2).handle(rootNode2); + verify(rootNodeHandler2).close(); + verify(rootNodeHandler3).handle(rootNode3); + verify(rootNodeHandler3).close(); + } + + @Test + public void fileGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException { --- End diff -- @mattyb149 lifecycle tests of the 3 different granularities using a doctored evtx file that has good chunks, bad chunk headers, and a malformed record > Processor to Parse .evtx files > ------------------------------ > > Key: NIFI-1975 > URL: https://issues.apache.org/jira/browse/NIFI-1975 > Project: Apache NiFi > Issue Type: Sub-task > Reporter: Bryan Rosander > > Windows event logs are stored in .evtx format as-of Windows Vista. If we > port the pure python implementation of an evtx parser at > https://github.com/williballenthin/python-evtx to Java, we should be able to > ingest those files in NiFi on any operating system -- This message was sent by Atlassian JIRA (v6.3.4#6332)