This is an automated email from the ASF dual-hosted git repository.
ashapkin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new f9bd7b7 IGNITE-15542 walTotalSize incorrectly reported when wal
archiving is turned off. (#9425)
f9bd7b7 is described below
commit f9bd7b73682ff9dd2865fdb559816367fc2d65b8
Author: akorensh <52174457+akore...@users.noreply.github.com>
AuthorDate: Wed Sep 22 09:24:16 2021 -0400
IGNITE-15542 walTotalSize incorrectly reported when wal archiving is turned
off. (#9425)
---
.../persistence/wal/FileWriteAheadLogManager.java | 6 +-
.../IgniteDataStorageMetricsSelfTest.java | 132 +++++++++++++++++++++
2 files changed, 136 insertions(+), 2 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 7e3fde2..bbab285 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -495,8 +495,10 @@ public class FileWriteAheadLogManager extends
GridCacheSharedManagerAdapter impl
for (File f : walWorkDir0.listFiles())
size += f.length();
- for (File f : walArchiveDir0.listFiles())
- size += f.length();
+ if (isArchiverEnabled()) {
+ for (File f : walArchiveDir0.listFiles())
+ size += f.length();
+ }
return size;
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
index 9a42ff1..f767fdd 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
@@ -17,18 +17,22 @@
package org.apache.ignite.internal.processors.cache.persistence;
+import java.io.File;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.DataStorageMetrics;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -43,9 +47,12 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import
org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
import
org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
+import
org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import
org.apache.ignite.internal.processors.cache.persistence.wal.SegmentRouter;
import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
+import org.apache.ignite.internal.processors.metric.impl.LongGauge;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.PAX;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -392,6 +399,131 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
}
/**
+ * Check whether WAL is reporting correct usage when archiving was not
needed.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWalTotalSizeWithoutArchivingPerfomed() throws Exception {
+ IgniteEx n = startGrid(
+ 0,
+ (Consumer<IgniteConfiguration>)cfg ->
cfg.getDataStorageConfiguration().setWalSegmentSize((int)(2 * U.MB))
+ );
+
+ n.cluster().state(ACTIVE);
+
+ populateCache(n);
+
+ disableWal(n, true);
+
+ checkWalArchiveAndTotalSize(n, true);
+ }
+
+ /**
+ * Check whether WAL is reporting correct usage when archiving is
performed.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWalTotalSizeWithArchive() throws Exception {
+ IgniteEx n = startGrid(
+ 0,
+ (Consumer<IgniteConfiguration>)cfg ->
cfg.getDataStorageConfiguration().setWalSegmentSize((int)(2 * U.MB))
+ );
+
+ n.cluster().state(ACTIVE);
+
+ while (walMgr(n).lastArchivedSegment() < 3)
+ n.cache("cache").put(ThreadLocalRandom.current().nextLong(), new
byte[(int)(32 * U.KB)]);
+
+ disableWal(n, true);
+
+ checkWalArchiveAndTotalSize(n, true);
+ }
+
+ /**
+ * Check whether Wal is reporting correct usage when WAL Archive is turned
off.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWalTotalSizeWithArchiveTurnedOff() throws Exception {
+ IgniteEx n = startGrid(
+ 0,
+ (Consumer<IgniteConfiguration>)cfg ->
cfg.getDataStorageConfiguration()
+
.setWalArchivePath(cfg.getDataStorageConfiguration().getWalPath()).setWalSegmentSize((int)(2
* U.MB))
+ );
+
+ n.cluster().state(ACTIVE);
+
+ populateCache(n);
+
+ disableWal(n, true);
+
+ checkWalArchiveAndTotalSize(n, false);
+ }
+
+ /**
+ * Populates a cache w/32 KB of data.
+ *
+ * @param igniteEx Node.
+ */
+ private void populateCache(IgniteEx igniteEx) {
+ for (int i = 0; i < 10; i++)
+
igniteEx.cache("cache").put(ThreadLocalRandom.current().nextLong(), new
byte[(int)(32 * U.KB)]);
+ }
+
+ /**
+ * Check the state of wal archive, and whether the total size of
+ * wal (and possibly wal archive) match what is expected.
+ *
+ * @param igniteEx Node.
+ * @param hasWalArchive Whether wal archiving is enabled.
+ * @throws Exception If failed.
+ */
+ private void checkWalArchiveAndTotalSize(IgniteEx igniteEx, boolean
hasWalArchive) throws Exception {
+ FileWriteAheadLogManager walMgr = walMgr(igniteEx);
+
+ SegmentRouter router = walMgr.getSegmentRouter();
+
+ assertEquals(router.hasArchive(), hasWalArchive);
+
+ //Wait to avoid race condition where new segments(and corresponding
.tmp files) are created after totalSize has been calculated.
+ if (router.hasArchive()) {
+ int expWalWorkSegements =
igniteEx.configuration().getDataStorageConfiguration().getWalSegments();
+
+ assertTrue(waitForCondition(() ->
walFiles(router.getWalWorkDir()).length == expWalWorkSegements, 3000l));
+
+ assertTrue(waitForCondition(() -> walMgr.lastArchivedSegment() ==
walMgr.currentSegment() - 1, 3000l));
+ }
+
+ long totalSize = walMgr.totalSize(walFiles(router.getWalWorkDir()));
+
+ if (router.hasArchive())
+ totalSize += walMgr.totalSize(walFiles(router.getWalArchiveDir()));
+
+ assertEquals(totalSize,
dbMgr(igniteEx).persistentStoreMetrics().getWalTotalSize());
+ assertEquals(totalSize, dsMetricsMXBean(igniteEx).getWalTotalSize());
+ assertEquals(totalSize,
((LongGauge)dsMetricRegistry(igniteEx).findMetric("WalTotalSize")).value());
+ }
+
+ /**
+ * List of all relevant wal files descriptors in a given directory.
+ *
+ * @param filesDir Directory where the wal files are located.
+ * @return List of relevant file descriptors
+ * @throws IgniteException If failed.
+ */
+ private FileDescriptor[] walFiles(final File filesDir) throws
IgniteException {
+ try {
+ return FileWriteAheadLogManager.loadFileDescriptors(filesDir);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
*
*/
static class Person implements Serializable {
