mattrpav commented on code in PR #2233: URL: https://github.com/apache/activemq/pull/2233#discussion_r3962466098
########## activemq-kahadb-store/src/test/java/org/apache/activemq/store/kahadb/disk/page/PageFileCompactionTest.java: ########## @@ -0,0 +1,385 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.store.kahadb.disk.page; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import org.apache.activemq.store.kahadb.disk.page.PageFile.PageFileCompactionStrategy; +import org.apache.activemq.store.kahadb.disk.util.StringMarshaller; +import org.apache.activemq.util.Wait; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class PageFileCompactionTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private PageFile pf; + + @After + public void tearDown() throws Exception { + if (pf != null) { + try { + pf.unload(); + } catch (Exception ignored) { + // ignore + } + } + } + + // min .1 + // max .3 + @Test + public void testFreeAll() throws IOException { + pf = new PageFile(tempFolder.newFolder(), "pagefile"); + pf.setCompactionStrategy(PageFileCompactionStrategy.TRUNCATION); + pf.load(); + + // write 100 pages with some test data + writePages(pf, 100, true); + + // verify page file properly allocated for all 100 pages + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // compaction shouldn't do anything + pf.compact(); + + // file length shouldn't have changed + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // free the first 50 pages, so 50% will now be marked as free + freePages(pf, 0, 50, false); + + // Compaction is not able to truncate because the free pages + // are only at the front + pf.compact(); + + // file length shouldn't have changed + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // free the remaining pages, but do not flush + freePages(pf, 50, 100, false); + // compaction should not do anything becuase the free pages + // were not yet flushed + pf.compact(); + + // file length shouldn't have changed + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // now we flush, compaction should finally truncate + pf.flush(); + pf.compact(); + + // All 100 pages were free, 10 should be left because of the 10% min ratio + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(10), pf.getDiskSize()); + assertEquals(10, pf.getFreePageCount()); + } + + @Test + public void testUnCleanShutdownRecoveryFile() throws Exception { + var tmp = tempFolder.newFolder(); + pf = new PageFile(tmp, "pagefile"); + pf.setCompactionStrategy(PageFileCompactionStrategy.TRUNCATION); + pf.load(); + + // write 100 pages with test data + writePages(pf, 100, true); + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // free 30 pages (the threshold) so we compact + freePages(pf, 70, 100, true); + assertEquals(30, pf.getFreePageCount()); + assertEquals(100, pf.getPageCount()); + + pf.compact(); + + // we should have truncated 20 pages + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(80), pf.getDiskSize()); + assertEquals(10, pf.getFreePageCount()); + assertEquals(80, pf.getPageCount()); + + // At this point the PageFile was not closed and shut down properly + // so the free page list wasn't persisted, etc. + + // The recovery file contains the last batch of writes if enabled, so + // in this case the last thing written was to free 30 pages so + // loading will bring back and reallocate the 30 free pages bringing + // the file back to 100 + PageFile pf2 = new PageFile(tmp, "pagefile"); + pf2.load(); + // free page recovery is async + assertTrue(Wait.waitFor(() -> pf2.recoveredFreeList.get() != null, 1000, 1)); + + // free page pages won't be recovered until we flush so all 100 + // pages should still be in use + assertEquals(0, pf2.getFreePageCount()); + assertEquals(100, pf2.getPageCount()); + assertEquals(pf2.toOffset(100), pf2.getDiskSize()); + + // flush merges recovered free pages and writes them again + pf2.flush(); + + // restored back to before compaction + assertEquals(pf2.getFile().length(), pf2.getDiskSize()); + assertEquals(pf2.toOffset(100), pf2.getDiskSize()); + assertEquals(30, pf2.getFreePageCount()); + assertEquals(100, pf2.getPageCount()); + } + + @Test + public void testUnCleanShutdownNoRecoveryFile() throws Exception { + var tmp = tempFolder.newFolder(); + pf = new PageFile(tmp, "pagefile"); + pf.setCompactionStrategy(PageFileCompactionStrategy.TRUNCATION); + // disable recovery file + pf.setEnableRecoveryFile(false); + pf.load(); + + // write 100 pages with test data + writePages(pf, 100, true); + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(100), pf.getDiskSize()); + + // free 30 pages (the threshold) so we compact + freePages(pf, 70, 100, true); + assertEquals(30, pf.getFreePageCount()); + assertEquals(100, pf.getPageCount()); + + pf.compact(); + + // we should have truncated 20 pages + assertEquals(pf.getFile().length(), pf.getDiskSize()); + assertEquals(pf.toOffset(80), pf.getDiskSize()); + assertEquals(10, pf.getFreePageCount()); + assertEquals(80, pf.getPageCount()); + + // At this point the PageFile was not closed and shut down properly + // so the free page list wasn't persisted, etc. On load + // 80 pages come back (none free) because we truncated 20 + + // The recovery file is not active + PageFile pf2 = new PageFile(tmp, "pagefile"); + pf2.load(); + pf2.flush(); + + // restored back to before compaction - the 20 pages were truncated + // and won't come back and are lost without a recovery file + assertEquals(pf2.getFile().length(), pf2.getDiskSize()); + assertEquals(pf2.toOffset(80), pf2.getDiskSize()); + assertEquals(0, pf2.getFreePageCount()); + assertEquals(80, pf2.getPageCount()); + + + + // TODO we also want to test with recovery file still enabled, but Review Comment: Address the TODO now or in a follow on? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected] For further information, visit: https://activemq.apache.org/contact
