exceptionfactory commented on code in PR #8152: URL: https://github.com/apache/nifi/pull/8152#discussion_r1479945239
########## nifi-nar-bundles/nifi-questdb-bundle/nifi-questdb/src/test/java/org/apache/nifi/questdb/embedded/EmbeddedDatabaseManagerTest.java: ########## @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.questdb.embedded; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.nifi.questdb.Client; +import org.apache.nifi.questdb.DatabaseException; +import org.apache.nifi.questdb.DatabaseManager; +import org.apache.nifi.questdb.mapping.RequestMapping; +import org.apache.nifi.questdb.rollover.RolloverStrategy; +import org.apache.nifi.questdb.util.Event; +import org.apache.nifi.questdb.util.QuestDbTestUtil; +import org.apache.nifi.util.file.FileUtils; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.StreamSupport; + +import static org.apache.nifi.questdb.util.QuestDbTestUtil.CREATE_EVENT2_TABLE; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.CREATE_EVENT_TABLE; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.EVENT2_TABLE_NAME; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.EVENT_TABLE_NAME; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.SELECT_QUERY; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.getRandomTestData; +import static org.apache.nifi.questdb.util.QuestDbTestUtil.getTestData; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EmbeddedDatabaseManagerTest extends EmbeddedQuestDbTest { + private static final Logger LOGGER = LoggerFactory.getLogger(EmbeddedDatabaseManagerTest.class); + private static final int DAYS_TO_KEEP_EVENT = 1; + private static final String NON_EXISTING_PLACE = SystemUtils.IS_OS_WINDOWS ? "T:/nonExistingPlace" : "/nonExistingPlace"; + + @Test + public void testAcquiringWithoutInitialization() { + final EmbeddedDatabaseManager testSubject = new EmbeddedDatabaseManager(new SimpleEmbeddedDatabaseManagerContext()); + assertThrows(IllegalStateException.class, () -> testSubject.acquireClient()); + } + + @Test + public void testHappyPath() throws DatabaseException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject = getTestSubject(); + assertDatabaseFolderIsNotEmpty(); + + final Client client = testSubject.acquireClient(); + + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + final List<Event> result = client.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + + assertIterableEquals(testData, result); + + testSubject.close(); + + // Even if the client itself is not connected, manager prevents client to reach database after closing + assertFalse(client.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)).iterator().hasNext()); + } + + @Test + public void testRollover() throws DatabaseException, InterruptedException { + final List<Event> testData = new ArrayList<>(); + testData.add(new Event(Instant.now().minus((DAYS_TO_KEEP_EVENT + 1), ChronoUnit.DAYS), "A", 1)); + testData.add(new Event(Instant.now(), "B", 2)); + final DatabaseManager testSubject = getTestSubject(); + + final Client client = testSubject.acquireClient(); + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + + // The rollover runs in every 5 seconds + Thread.sleep(TimeUnit.SECONDS.toMillis(6)); + + final List<Event> result = client.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + testSubject.close(); + + assertEquals(1, result.size()); + } + + @Test + public void testParallelClientsOnSameThread() throws DatabaseException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject = getTestSubject(); + + final Client client1 = testSubject.acquireClient(); + client1.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + final List<Event> result1 = client1.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + + final Client client2 = testSubject.acquireClient(); + final List<Event> result2 = client2.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + + testSubject.close(); + assertEquals(3, result2.size()); + assertIterableEquals(result1, result2); + } + + @Test + public void testParallelClientsDifferentThread() throws InterruptedException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject = getTestSubject(); + final CountDownLatch step1 = new CountDownLatch(1); + final CountDownLatch step2 = new CountDownLatch(1); + final AtomicReference<List<Event>> result1 = new AtomicReference<>(); + final AtomicReference<List<Event>> result2 = new AtomicReference<>(); + + new Thread(() -> { + try { + final Client client1 = testSubject.acquireClient(); + client1.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + result1.set(client1.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING))); + step1.countDown(); + } catch (final DatabaseException e) { + throw new RuntimeException(e); + } + }).start(); + + new Thread(() -> { + try { + step1.await(); + final Client client2 = testSubject.acquireClient(); + result2.set(client2.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING))); + step2.countDown(); + } catch (final DatabaseException | InterruptedException e) { + throw new RuntimeException(e); + } + }).start(); + + step2.await(); + + testSubject.close(); + assertEquals(3, result1.get().size()); + assertIterableEquals(result1.get(), result2.get()); + } + + @Test + public void testContactingToDatabaseWithDifferentManager() throws DatabaseException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject1 = getTestSubject(); + + final Client client1 = testSubject1.acquireClient(); + client1.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + final List<Event> result1 = client1.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + client1.disconnect(); + testSubject1.close(); + + assertDatabaseFolderIsNotEmpty(); + + final DatabaseManager testSubject2 = getTestSubject(); + final Client client2 = testSubject2.acquireClient(); + final List<Event> result2 = client2.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + client2.disconnect(); + testSubject2.close(); + + assertIterableEquals(result1, result2); + } + + @Test + public void testDatabaseRestorationAfterLostDatabase() throws DatabaseException, IOException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject = getTestSubject(); + final Client client = testSubject.acquireClient(); + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + + FileUtils.deleteFilesInDir(testDbPathDirectory.toFile(), (dir, name) -> true, LOGGER, true, true); + + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + final List<Event> result = client.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + + testSubject.close(); + + // Ensuring that not the fallback client answers + assertEquals(3, result.size()); + } + + @Test + public void testDatabaseRestorationAfterLosingTableFiles() throws DatabaseException, IOException { + final List<Event> testData = getTestData(); + final DatabaseManager testSubject = getTestSubject(); + final Client client = testSubject.acquireClient(); + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + + final File eventTableDirectory = new File(testDbPathDirectory.toFile(), "event"); + FileUtils.deleteFilesInDir(eventTableDirectory, (dir, name) -> true, LOGGER, true, true); + FileUtils.deleteFile(eventTableDirectory, LOGGER); + + client.insert(EVENT_TABLE_NAME, QuestDbTestUtil.getEventTableDataSource(testData)); + final List<Event> result = client.query(SELECT_QUERY, RequestMapping.getResultProcessor(QuestDbTestUtil.EVENT_TABLE_REQUEST_MAPPING)); + + testDbPathDirectory.toFile().list((dir, name) -> dir.isDirectory()); + + testSubject.close(); + + // Ensuring that not the fallback client answers + assertEquals(3, result.size()); + } + + @Test + @DisabledOnOs(OS.WINDOWS) + // This testcase cannot be reproduced under Windows using Junit + public void testDatabaseRestorationAfterCorruptedFiles() throws DatabaseException, IOException { Review Comment: The latest updates in https://github.com/apache/nifi/pull/8152/commits/e33dbe91571ef3373f356f028641f366cb4bb6c7 now clear out the `questDbBackup` directory, but the directory itself still remains. It looks like adding one more step to delete the directory itself should resolve this particular issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
