[ https://issues.apache.org/jira/browse/HADOOP-18258?focusedWorklogId=783052&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-783052 ]
ASF GitHub Bot logged work on HADOOP-18258: ------------------------------------------- Author: ASF GitHub Bot Created on: 20/Jun/22 16:08 Start Date: 20/Jun/22 16:08 Worklog Time Spent: 10m Work Description: sravanigadey commented on code in PR #4383: URL: https://github.com/apache/hadoop/pull/4383#discussion_r901828806 ########## hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestS3AAuditLogMerger.java: ########## @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.audit; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * MergerTest will implement different tests on Merger class methods. + */ +public class TestS3AAuditLogMerger { + + private final Logger logger = LoggerFactory.getLogger(TestS3AAuditLogMerger.class); + + private final S3AAuditLogMerger s3AAuditLogMerger = new S3AAuditLogMerger(); + + /** + * sample directories and files to test. + */ + private final File auditLogFile = new File("AuditLogFile"); + private final File sampleDirectory = new File("sampleFilesDirectory"); + private final File emptyDirectory = new File("emptyFilesDirectory"); + private final File firstSampleFile = + new File("sampleFilesDirectory", "sampleFile1.txt"); + private final File secondSampleFile = + new File("sampleFilesDirectory", "sampleFile2.txt"); + private final File thirdSampleFile = + new File("sampleFilesDirectory", "sampleFile3.txt"); + + /** + * creates the sample directories and files before each test. + * + * @throws IOException on failure + */ + @Before + public void setUp() throws IOException { + boolean sampleDirCreation = sampleDirectory.mkdir(); + boolean emptyDirCreation = emptyDirectory.mkdir(); + if (sampleDirCreation && emptyDirCreation) { + try (FileWriter fw = new FileWriter(firstSampleFile); + FileWriter fw1 = new FileWriter(secondSampleFile); + FileWriter fw2 = new FileWriter(thirdSampleFile)) { + fw.write("abcd"); + fw1.write("efgh"); + fw2.write("ijkl"); + } + } + } + + /** + * mergeFilesTest() will test the mergeFiles() method in Merger class. + * by passing a sample directory which contains files with some content in it + * and checks if files in a directory are merged into single file + * + * @throws IOException on any failure Review Comment: removed @throws ########## hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestS3AAuditLogMerger.java: ########## @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.audit; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * MergerTest will implement different tests on Merger class methods. + */ +public class TestS3AAuditLogMerger { + + private final Logger logger = LoggerFactory.getLogger(TestS3AAuditLogMerger.class); + + private final S3AAuditLogMerger s3AAuditLogMerger = new S3AAuditLogMerger(); + + /** + * sample directories and files to test. + */ + private final File auditLogFile = new File("AuditLogFile"); + private final File sampleDirectory = new File("sampleFilesDirectory"); + private final File emptyDirectory = new File("emptyFilesDirectory"); + private final File firstSampleFile = + new File("sampleFilesDirectory", "sampleFile1.txt"); + private final File secondSampleFile = + new File("sampleFilesDirectory", "sampleFile2.txt"); + private final File thirdSampleFile = + new File("sampleFilesDirectory", "sampleFile3.txt"); + + /** + * creates the sample directories and files before each test. + * + * @throws IOException on failure + */ + @Before + public void setUp() throws IOException { + boolean sampleDirCreation = sampleDirectory.mkdir(); + boolean emptyDirCreation = emptyDirectory.mkdir(); + if (sampleDirCreation && emptyDirCreation) { + try (FileWriter fw = new FileWriter(firstSampleFile); + FileWriter fw1 = new FileWriter(secondSampleFile); + FileWriter fw2 = new FileWriter(thirdSampleFile)) { + fw.write("abcd"); + fw1.write("efgh"); + fw2.write("ijkl"); + } + } + } + + /** + * mergeFilesTest() will test the mergeFiles() method in Merger class. + * by passing a sample directory which contains files with some content in it + * and checks if files in a directory are merged into single file + * + * @throws IOException on any failure Review Comment: removed @throws in tests Issue Time Tracking ------------------- Worklog Id: (was: 783052) Time Spent: 3h 10m (was: 3h) > Merging of S3A Audit Logs > ------------------------- > > Key: HADOOP-18258 > URL: https://issues.apache.org/jira/browse/HADOOP-18258 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/s3 > Reporter: Sravani Gadey > Assignee: Sravani Gadey > Priority: Major > Labels: pull-request-available > Time Spent: 3h 10m > Remaining Estimate: 0h > > Merging audit log files containing huge number of audit logs collected from a > job like Hive or Spark job containing various S3 requests like list, head, > get and put requests. -- This message was sent by Atlassian Jira (v8.20.7#820007) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org