This is an automated email from the ASF dual-hosted git repository. alexpl pushed a commit to branch ignite-2.14 in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-2.14 by this push: new bdd7cfb2843 IGNITE-13726 Add system view for count of hot/cold pages in page-memory - Fixes #8474. bdd7cfb2843 is described below commit bdd7cfb2843f827930fd2847ee2167ac3c8245cc Author: Aleksey Plekhanov <plehanov.a...@gmail.com> AuthorDate: Wed Aug 31 11:18:07 2022 +0300 IGNITE-13726 Add system view for count of hot/cold pages in page-memory - Fixes #8474. Signed-off-by: Aleksey Plekhanov <plehanov.a...@gmail.com> (cherry picked from commit 8f41a94f8ac8e4ccb8a2f5e33a204b88e17f8559) --- .../ignite/jdbc/thin/JdbcThinMetadataSelfTest.java | 9 +- .../SystemViewRowAttributeWalkerGenerator.java | 2 + .../apache/ignite/util/SystemViewCommandTest.java | 3 +- .../walker/PagesTimestampHistogramViewWalker.java | 51 +++ .../cache/persistence/DataRegionMetricsImpl.java | 73 +++++ .../IgniteCacheDatabaseSharedManager.java | 17 + .../cache/persistence/pagemem/PageHeader.java | 19 +- .../cache/persistence/pagemem/PageMemoryImpl.java | 59 +++- .../internal/processors/metric/AbstractMetric.java | 2 +- .../metric/ConfigurableHistogramMetric.java | 28 ++ .../processors/metric/GridMetricManager.java | 15 +- .../metric/impl/HistogramMetricImpl.java | 9 +- .../metric/impl/PeriodicHistogramMetricImpl.java | 327 +++++++++++++++++++ .../view/PagesTimestampHistogramView.java | 75 +++++ .../internal/metric/MetricsConfigurationTest.java | 45 +++ .../metric/PeriodicHistogramMetricImplTest.java | 359 +++++++++++++++++++++ .../ignite/internal/metric/SystemViewSelfTest.java | 209 +++++++++++- .../persistence/pagemem/PageMemoryImplTest.java | 1 + .../ignite/internal/util/GridTestClockTimer.java | 31 +- .../ignite/testsuites/IgniteBasicTestSuite2.java | 4 +- .../cache/metric/SqlViewExporterSpiTest.java | 42 ++- 21 files changed, 1350 insertions(+), 30 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java index c9cf9bd225e..f6e2ef87df1 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java @@ -457,7 +457,8 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest { "SYS.STATISTICS_LOCAL_DATA", "SYS.STATISTICS_GLOBAL_DATA", "SYS.STATISTICS_PARTITION_DATA", - "SYS.STATISTICS_CONFIGURATION" + "SYS.STATISTICS_CONFIGURATION", + "SYS.PAGES_TIMESTAMP_HISTOGRAM" )) ); } @@ -1132,7 +1133,11 @@ public class JdbcThinMetadataSelfTest extends JdbcThinAbstractSelfTest { "SYS.STATISTICS_GLOBAL_DATA.TOTAL.null.19", "SYS.STATISTICS_GLOBAL_DATA.SIZE.null.10", "SYS.STATISTICS_GLOBAL_DATA.VERSION.null.19", - "SYS.STATISTICS_GLOBAL_DATA.LAST_UPDATE_TIME.null.2147483647" + "SYS.STATISTICS_GLOBAL_DATA.LAST_UPDATE_TIME.null.2147483647", + "SYS.PAGES_TIMESTAMP_HISTOGRAM.DATA_REGION_NAME.null.2147483647", + "SYS.PAGES_TIMESTAMP_HISTOGRAM.INTERVAL_START.null.26.6", + "SYS.PAGES_TIMESTAMP_HISTOGRAM.INTERVAL_END.null.26.6", + "SYS.PAGES_TIMESTAMP_HISTOGRAM.PAGES_COUNT.null.19" )); Assert.assertEquals(expectedCols, actualSystemCols); diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java index be028aaccfb..850ce8ffd9f 100644 --- a/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java +++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java @@ -57,6 +57,7 @@ import org.apache.ignite.spi.systemview.view.MetricsView; import org.apache.ignite.spi.systemview.view.NodeAttributeView; import org.apache.ignite.spi.systemview.view.NodeMetricsView; import org.apache.ignite.spi.systemview.view.PagesListView; +import org.apache.ignite.spi.systemview.view.PagesTimestampHistogramView; import org.apache.ignite.spi.systemview.view.PartitionStateView; import org.apache.ignite.spi.systemview.view.ScanQueryView; import org.apache.ignite.spi.systemview.view.ServiceView; @@ -145,6 +146,7 @@ public class SystemViewRowAttributeWalkerGenerator { gen.generateAndWrite(CacheGroupIoView.class, DFLT_SRC_DIR); gen.generateAndWrite(SnapshotView.class, DFLT_SRC_DIR); gen.generateAndWrite(MetricsView.class, DFLT_SRC_DIR); + gen.generateAndWrite(PagesTimestampHistogramView.class, DFLT_SRC_DIR); gen.generateAndWrite(SqlSchemaView.class, INDEXING_SRC_DIR); gen.generateAndWrite(SqlTableView.class, INDEXING_SRC_DIR); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java index 6e27942fa3a..230565d864c 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java @@ -461,7 +461,8 @@ public class SystemViewCommandTest extends GridCommandHandlerClusterByClassAbstr "DS_REENTRANTLOCKS", "DS_SETS", "DS_SEMAPHORES", - "DS_QUEUES" + "DS_QUEUES", + "PAGES_TIMESTAMP_HISTOGRAM" )); Set<String> viewNames = new TreeSet<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/PagesTimestampHistogramViewWalker.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/PagesTimestampHistogramViewWalker.java new file mode 100644 index 00000000000..85d32c7800d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/PagesTimestampHistogramViewWalker.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.systemview.walker; + +import java.util.Date; +import org.apache.ignite.spi.systemview.view.PagesTimestampHistogramView; +import org.apache.ignite.spi.systemview.view.SystemViewRowAttributeWalker; + +/** + * Generated by {@code org.apache.ignite.codegen.SystemViewRowAttributeWalkerGenerator}. + * {@link PagesTimestampHistogramView} attributes walker. + * + * @see PagesTimestampHistogramView + */ +public class PagesTimestampHistogramViewWalker implements SystemViewRowAttributeWalker<PagesTimestampHistogramView> { + /** {@inheritDoc} */ + @Override public void visitAll(AttributeVisitor v) { + v.accept(0, "dataRegionName", String.class); + v.accept(1, "intervalStart", Date.class); + v.accept(2, "intervalEnd", Date.class); + v.accept(3, "pagesCount", long.class); + } + + /** {@inheritDoc} */ + @Override public void visitAll(PagesTimestampHistogramView row, AttributeWithValueVisitor v) { + v.accept(0, "dataRegionName", String.class, row.dataRegionName()); + v.accept(1, "intervalStart", Date.class, row.intervalStart()); + v.accept(2, "intervalEnd", Date.class, row.intervalEnd()); + v.acceptLong(3, "pagesCount", row.pagesCount()); + } + + /** {@inheritDoc} */ + @Override public int count() { + return 4; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java index 1bf218d7f0e..188ed968874 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java @@ -16,6 +16,10 @@ */ package org.apache.ignite.internal.processors.cache.persistence; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Optional; import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataRegionMetricsProvider; @@ -30,10 +34,13 @@ import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; import org.apache.ignite.internal.processors.metric.impl.LongAdderWithDelegateMetric; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.metric.impl.PeriodicHistogramMetricImpl; import org.apache.ignite.internal.util.collection.IntHashMap; import org.apache.ignite.internal.util.collection.IntMap; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.mxbean.MetricsMxBean; +import org.apache.ignite.spi.systemview.view.PagesTimestampHistogramView; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; @@ -160,6 +167,9 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { /** Time interval (in milliseconds) when allocations/evictions are counted to calculate rate. */ private volatile long rateTimeInterval; + /** Histogram of cold/hot pages. */ + private final PeriodicHistogramMetricImpl pageTsHistogram; + /** * Same as {@link #DataRegionMetricsImpl(DataRegionConfiguration, GridKernalContext, DataRegionMetricsProvider)} * but uses a no-op implementation for the {@link DataRegionMetricsProvider}. @@ -257,6 +267,20 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { mreg.longMetric("MaxSize", "Maximum memory region size in bytes defined by its data region.") .value(dataRegionCfg.getMaxSize()); + if (persistenceEnabled) { + // Reserve 1 sec, page ts can be slightly lower than currentTimeMillis, due to applied to ts mask. This + // reservation mainly affects only tests (we can check buckets more predictevely). + long startTs = U.currentTimeMillis() - 1000L; + String name = MetricUtils.metricName(mreg.name(), "PageTimestampHistogram"); + String desc = "Histogram of pages last access time"; + + pageTsHistogram = new PeriodicHistogramMetricImpl(startTs, name, desc); + + mreg.register(pageTsHistogram); + } + else + pageTsHistogram = null; + dataRegionPageMetrics = PageMetricsImpl.builder(mreg) .totalPagesCallback(new LongAdderWithDelegateMetric.Delegate() { @Override public void increment() { @@ -619,6 +643,9 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { */ public void enableMetrics() { metricsEnabled = true; + + if (pageTsHistogram != null) + pageTsHistogram.reset(getPhysicalMemoryPages()); } /** @@ -626,6 +653,9 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { */ public void disableMetrics() { metricsEnabled = false; + + if (pageTsHistogram != null) + pageTsHistogram.reset(0); } /** @@ -739,4 +769,47 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { if (metricsEnabled) totalThrottlingTime.add(time); } + + /** + * Increment count of pages with given last access time. + * + * @param ts Last access timestamp. + */ + public void incrementPagesWithTimestamp(long ts) { + if (metricsEnabled && pageTsHistogram != null) + pageTsHistogram.increment(ts); + } + + /** + * Decrement count of pages with given last access time. + * + * @param ts Last access timestamp. + */ + public void decrementPagesWithTimestamp(long ts) { + if (metricsEnabled && pageTsHistogram != null) + pageTsHistogram.decrement(ts); + } + + /** + * Creates pages timestamp histogram view. + */ + public Collection<PagesTimestampHistogramView> pagesTimestampHistogramView() { + if (!metricsEnabled || pageTsHistogram == null) + return Collections.emptyList(); + + IgniteBiTuple<long[], long[]> hist = pageTsHistogram.histogram(); + + long[] bounds = hist.get1(); + long[] vals = hist.get2(); + + List<PagesTimestampHistogramView> list = new ArrayList<>(vals.length); + + for (int i = 0; i < vals.length - 1; i++) + list.add(new PagesTimestampHistogramView(getName(), bounds[i], bounds[i + 1], vals[i])); + + list.add(new PagesTimestampHistogramView(getName(), bounds[vals.length - 1], + U.currentTimeMillis(), vals[vals.length - 1])); + + return list; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 75ba6cce6c0..8cf730a6a33 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -49,6 +49,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.managers.systemview.walker.PagesListViewWalker; +import org.apache.ignite.internal.managers.systemview.walker.PagesTimestampHistogramViewWalker; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; @@ -82,6 +83,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.warmup.WarmUpStrategy; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.TimeBag; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.LT; @@ -128,6 +130,12 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** System view description for page lists. */ public static final String DATA_REGION_PAGE_LIST_VIEW_DESC = "Data region page lists"; + /** System view name for pages timestamp histogram. */ + public static final String PAGE_TS_HISTOGRAM_VIEW = "pagesTimestampHistogram"; + + /** System view description for pages timestamp histogram. */ + public static final String PAGE_TS_HISTOGRAM_VIEW_DESC = "Data region pages timestamp histogram"; + /** Minimum size of memory chunk */ private static final long MIN_PAGE_MEMORY_SIZE = 10L * 1024 * 1024; @@ -230,6 +238,15 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap }, Function.identity() ); + + cctx.kernalContext().systemView().registerInnerCollectionView( + PAGE_TS_HISTOGRAM_VIEW, + PAGE_TS_HISTOGRAM_VIEW_DESC, + new PagesTimestampHistogramViewWalker(), + F.viewReadOnly(dataRegions(), DataRegion::metrics), + DataRegionMetricsImpl::pagesTimestampHistogramView, + (pageMemory, view) -> view + ); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageHeader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageHeader.java index 185878a1658..2fcd6bd3087 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageHeader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageHeader.java @@ -27,6 +27,9 @@ class PageHeader { /** */ public static final long PAGE_MARKER = 0x0000000000000001L; + /** */ + public static final long TIMESTAMP_MASK = 0xFFFFFFFFFFFFFF00L; + /** Dirty flag. */ private static final long DIRTY_FLAG = 0x0100000000000000L; @@ -169,11 +172,19 @@ class PageHeader { * Volatile write for current timestamp to page in {@code absAddr} address. * * @param absPtr Absolute page address. + * @return Old page timestamp value. */ - public static void writeTimestamp(final long absPtr, long tstamp) { - tstamp &= 0xFFFFFFFFFFFFFF00L; + public static long writeTimestamp(final long absPtr, long tstamp) { + long oldTs; + + tstamp &= TIMESTAMP_MASK; + tstamp |= 0x01L; + + do { + oldTs = GridUnsafe.getLong(absPtr); + } while (!GridUnsafe.compareAndSwapLong(null, absPtr, oldTs, tstamp)); - GridUnsafe.putLongVolatile(null, absPtr, tstamp | 0x01); + return oldTs & TIMESTAMP_MASK; } /** @@ -186,7 +197,7 @@ class PageHeader { long markerAndTs = GridUnsafe.getLong(absPtr); // Clear last byte as it is occupied by page marker. - return markerAndTs & ~0xFF; + return markerAndTs & TIMESTAMP_MASK; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 7d709c618cb..b487ec7c688 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -574,24 +574,33 @@ public class PageMemoryImpl implements PageMemoryEx { OUTDATED_REL_PTR ); + boolean pageReplaced = false; + if (relPtr == OUTDATED_REL_PTR) { relPtr = seg.refreshOutdatedPage(grpId, pageId, false); seg.pageReplacementPolicy.onRemove(relPtr); + + pageReplaced = true; } if (relPtr == INVALID_REL_PTR) relPtr = seg.borrowOrAllocateFreePage(pageId); - if (relPtr == INVALID_REL_PTR) + if (relPtr == INVALID_REL_PTR) { relPtr = seg.removePageForReplacement(); + pageReplaced = true; + } + long absPtr = seg.absolute(relPtr); GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0); PageHeader.fullPageId(absPtr, fullId); - PageHeader.writeTimestamp(absPtr, U.currentTimeMillis()); + + touchPage(absPtr, pageReplaced); + rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId)); assert PageIO.getCrc(absPtr + PAGE_OVERHEAD) == 0; //TODO GG-11480 @@ -800,13 +809,19 @@ public class PageMemoryImpl implements PageMemoryEx { if (pageAllocated != null) pageAllocated.set(true); - if (relPtr == INVALID_REL_PTR) + boolean pageReplaced = false; + + if (relPtr == INVALID_REL_PTR) { relPtr = seg.removePageForReplacement(); + pageReplaced = true; + } + absPtr = seg.absolute(relPtr); PageHeader.fullPageId(absPtr, fullId); - PageHeader.writeTimestamp(absPtr, U.currentTimeMillis()); + + touchPage(absPtr, pageReplaced); assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + @@ -861,7 +876,9 @@ public class PageMemoryImpl implements PageMemoryEx { GridUnsafe.setMemory(pageAddr, pageSize(), (byte)0); PageHeader.fullPageId(absPtr, fullId); - PageHeader.writeTimestamp(absPtr, U.currentTimeMillis()); + + touchPage(absPtr, true); + PageIO.setPageId(pageAddr, pageId); assert !PageHeader.isAcquired(absPtr) : @@ -1241,6 +1258,8 @@ public class PageMemoryImpl implements PageMemoryEx { true ); + dataRegionMetrics.decrementPagesWithTimestamp(PageHeader.readTimestamp(seg.absolute(relPtr))); + seg.pageReplacementPolicy.onRemove(relPtr); seg.pool.releaseFreePage(relPtr); @@ -1464,7 +1483,7 @@ public class PageMemoryImpl implements PageMemoryEx { CountDownFuture completeFut = new CountDownFuture(segments.length); for (Segment seg : segments) { - Runnable clear = new ClearSegmentRunnable(seg, pred, cleanDirty, completeFut, pageSize()); + Runnable clear = new ClearSegmentRunnable(seg, dataRegionMetrics, pred, cleanDirty, completeFut, pageSize()); try { asyncRunner.execute(clear); @@ -1577,7 +1596,7 @@ public class PageMemoryImpl implements PageMemoryEx { return 0; if (touch) - PageHeader.writeTimestamp(absPtr, U.currentTimeMillis()); + touchPage(absPtr, true); assert PageIO.getCrc(absPtr + PAGE_OVERHEAD) == 0; //TODO GG-11480 @@ -1638,7 +1657,7 @@ public class PageMemoryImpl implements PageMemoryEx { * @return Pointer to the page write buffer. */ private long postWriteLockPage(long absPtr, FullPageId fullId) { - PageHeader.writeTimestamp(absPtr, U.currentTimeMillis()); + touchPage(absPtr, true); // Create a buffer copy if the page is scheduled for a checkpoint. if (isInCheckpoint(fullId) && PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR) { @@ -2491,6 +2510,23 @@ public class PageMemoryImpl implements PageMemoryEx { } } + /** + * Update timestamp for the page and reflect this change to the hot/cold pages histogram. + * + * @param absPtr Absolute pointer. + * @param pageExists Page already exists in page memory (histogram for old timestamp should be changed). + */ + private void touchPage(long absPtr, boolean pageExists) { + long newTs = U.currentTimeMillis(); + + long oldTs = PageHeader.writeTimestamp(absPtr, newTs); + + if (pageExists) + dataRegionMetrics.decrementPagesWithTimestamp(oldTs); + + dataRegionMetrics.incrementPagesWithTimestamp(newTs & PageHeader.TIMESTAMP_MASK); + } + /** * */ @@ -2510,6 +2546,9 @@ public class PageMemoryImpl implements PageMemoryEx { /** */ private final boolean rmvDirty; + /** */ + private final DataRegionMetricsImpl memMetrics; + /** * @param seg Segment. * @param clearPred Clear predicate for (cache group ID, page ID). @@ -2517,12 +2556,14 @@ public class PageMemoryImpl implements PageMemoryEx { */ private ClearSegmentRunnable( Segment seg, + DataRegionMetricsImpl memMetrics, LoadedPagesMap.KeyPredicate clearPred, boolean rmvDirty, CountDownFuture doneFut, int pageSize ) { this.seg = seg; + this.memMetrics = memMetrics; this.clearPred = clearPred; this.rmvDirty = rmvDirty; this.doneFut = doneFut; @@ -2558,6 +2599,8 @@ public class PageMemoryImpl implements PageMemoryEx { long absPtr = seg.pool.absolute(relPtr); + memMetrics.decrementPagesWithTimestamp(PageHeader.readTimestamp(absPtr)); + if (rmvDirty) { FullPageId fullId = PageHeader.fullPageId(absPtr); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/AbstractMetric.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/AbstractMetric.java index 7067307e803..c93322aa3a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/AbstractMetric.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/AbstractMetric.java @@ -34,7 +34,7 @@ public abstract class AbstractMetric implements Metric { * @param name Name. * @param desc Description. */ - public AbstractMetric(String name, String desc) { + protected AbstractMetric(String name, @Nullable String desc) { assert name != null; assert !name.isEmpty(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java new file mode 100644 index 00000000000..2a3d15ca16e --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.metric; + +import org.apache.ignite.spi.metric.HistogramMetric; + +/** + * Histogram metric with configurable bounds. + */ +public interface ConfigurableHistogramMetric extends HistogramMetric { + /** Sets bounds for this histogram. */ + public void bounds(long[] bounds); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java index ba4b1d9743e..78bb0a9e383 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java @@ -43,7 +43,6 @@ import org.apache.ignite.internal.processors.metastorage.DistributedMetastorageL import org.apache.ignite.internal.processors.metastorage.ReadableDistributedMetaStorage; import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.processors.metric.impl.DoubleMetricImpl; -import org.apache.ignite.internal.processors.metric.impl.HistogramMetricImpl; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; import org.apache.ignite.internal.util.future.GridCompoundFuture; @@ -460,7 +459,7 @@ public class GridMetricManager extends GridManagerAdapter<MetricExporterSpi> imp * * @param name Metric name. * @param rateTimeInterval New rateTimeInterval. - * @see HistogramMetricImpl#reset(long[]) + * @see HitRateMetric#reset(long) */ private void onHitRateConfigChanged(String name, @Nullable Long rateTimeInterval) { if (rateTimeInterval == null) @@ -486,12 +485,18 @@ public class GridMetricManager extends GridManagerAdapter<MetricExporterSpi> imp if (bounds == null) return; - HistogramMetricImpl m = find(name, HistogramMetricImpl.class); + ConfigurableHistogramMetric m = find(name, ConfigurableHistogramMetric.class); if (m == null) return; - m.reset(bounds); + try { + m.bounds(bounds); + } + catch (RuntimeException e) { + // Can't throw exceptions here since method is invoked by metastorage listener. + log.error("Error during histogram bounds reconfiguration", e); + } } /** @@ -522,7 +527,7 @@ public class GridMetricManager extends GridManagerAdapter<MetricExporterSpi> imp return null; } - if (!m.getClass().isAssignableFrom(type)) { + if (!type.isAssignableFrom(m.getClass())) { log.error("Metric '" + name + "' has wrong type[type=" + m.getClass().getSimpleName() + ']'); return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java index b174996538b..df2731c3273 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java @@ -20,14 +20,14 @@ package org.apache.ignite.internal.processors.metric.impl; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.ignite.internal.processors.metric.AbstractMetric; +import org.apache.ignite.internal.processors.metric.ConfigurableHistogramMetric; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.spi.metric.HistogramMetric; import org.jetbrains.annotations.Nullable; /** * Histogram metric implementation. */ -public class HistogramMetricImpl extends AbstractMetric implements HistogramMetric { +public class HistogramMetricImpl extends AbstractMetric implements ConfigurableHistogramMetric { /** Holder of measurements. */ private volatile HistogramHolder holder; @@ -83,6 +83,11 @@ public class HistogramMetricImpl extends AbstractMetric implements HistogramMetr holder = new HistogramHolder(bounds); } + /** {@inheritDoc} */ + @Override public void bounds(long[] bounds) { + reset(bounds); + } + /** {@inheritDoc} */ @Override public void reset() { reset(holder.bounds); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java new file mode 100644 index 00000000000..f412f0feaec --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.metric.impl; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; +import org.apache.ignite.internal.processors.metric.AbstractMetric; +import org.apache.ignite.internal.processors.metric.ConfigurableHistogramMetric; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.Nullable; + +/** + * Histogram to show count of items for each time interval with limited set of intervals. + * + * Count of items in interval can be incremented or decremented by timestamp. Items with timestamp below the first + * interval are moved into "out of bounds interval". Over time new intervals are added and old intervals are + * merged into "out of bounds interval" to maintain the same total count of intervals. + */ +@SuppressWarnings({"FieldAccessedSynchronizedAndUnsynchronized"}) +public class PeriodicHistogramMetricImpl extends AbstractMetric implements ConfigurableHistogramMetric { + /** Default buckets interval in milliseconds. */ + public static final long DFLT_BUCKETS_INTERVAL = 60L * 60 * 1000; // 60 mins. + + /** Default buckets count. */ + public static final int DFLT_BUCKETS_CNT = 24; + + /** Buckets interval in milliseconds. */ + private long bucketsInterval; + + /** Buckets count. */ + private int bucketsCnt; + + /** Starting point for bucket index calculation. */ + private volatile long startTs; + + /** Lower bound for values stored in buckets array (including). */ + private volatile long lowerBoundTs; + + /** Upper bound for values stored in buckets array (excluding). */ + private volatile long upperBoundTs; + + /** Out of bounds bucket. Contains count of items which have timestamp beyond lowerBoundTs. */ + private final AtomicLong outOfBoundsBucket = new AtomicLong(); + + /** Time of histogram creation. */ + private final long createTs = U.currentTimeMillis(); + + /** Buckets holder. */ + private volatile AtomicLongArray buckets; + + /** + * @param name Metric name. + * @param desc Metric description. + */ + public PeriodicHistogramMetricImpl(String name, @Nullable String desc) { + this(U.currentTimeMillis(), name, desc); + } + + /** + * @param startTs Starting point. + * @param name Metric name. + * @param desc Metric description. + */ + public PeriodicHistogramMetricImpl(long startTs, String name, @Nullable String desc) { + this(startTs, name, desc, DFLT_BUCKETS_INTERVAL, DFLT_BUCKETS_CNT); + } + + /** + * @param startTs Starting point. + * @param name Metric name. + * @param desc Metric description. + * @param bucketsInterval Buckets interval. + * @param bucketsCnt Buckets count. + */ + private PeriodicHistogramMetricImpl(long startTs, String name, @Nullable String desc, long bucketsInterval, int bucketsCnt) { + super(name, desc); + + reinit(bucketsInterval, bucketsCnt); + + this.startTs = startTs; + lowerBoundTs = startTs; + upperBoundTs = startTs + bucketsInterval; + } + + /** {@inheritDoc} */ + @Override public long[] bounds() { + long[] boundsIncludingFirst = histogram().get1(); + + // Exclude lower bound as it required by methods contract. + return Arrays.copyOfRange(boundsIncludingFirst, 1, boundsIncludingFirst.length); + } + + /** {@inheritDoc} */ + @Override public void bounds(long[] bounds) { + A.notNull(bounds, "bounds"); + A.ensure(bounds.length > 1, "bounds.length > 1"); + A.ensure(bounds[0] < bounds[1], "bounds[0] < bounds[1]"); + + // We need only interval between bounds and count of buckets, skip all values except first 2. + reinit(bounds[1] - bounds[0], bounds.length); + } + + /** {@inheritDoc} */ + @Override public long[] value() { + return histogram().get2(); + } + + /** {@inheritDoc} */ + @Override public Class<long[]> type() { + return long[].class; + } + + /** + * @param bucketsInterval Buckets interval. + * @param bucketsCnt Buckets count. + */ + public synchronized void reinit(long bucketsInterval, int bucketsCnt) { + startTs = U.currentTimeMillis(); + lowerBoundTs = startTs; + upperBoundTs = startTs + bucketsInterval; + + this.bucketsInterval = bucketsInterval; + this.bucketsCnt = bucketsCnt + 1; // One extra (dummy) bucket is reserved to deal with races. + + AtomicLongArray oldBuckets = buckets; + + buckets = new AtomicLongArray(this.bucketsCnt); + + if (oldBuckets != null) { + for (int i = 0; i < oldBuckets.length(); i++) + outOfBoundsBucket.addAndGet(oldBuckets.getAndSet(i, 0)); + } + } + + /** + * @param itemsCnt Total items count. + */ + public synchronized void reset(long itemsCnt) { + reinit(bucketsInterval, bucketsCnt); + + outOfBoundsBucket.set(itemsCnt); + } + + /** + * Increment count of items in interval by timestamp. + */ + public void increment(long ts) { + add(ts, 1); + } + + /** + * Decrement count of items in interval by timestamp. + */ + public void decrement(long ts) { + add(ts, -1); + } + + /** + * Gets histogram. + * + * @return Tuple, where first item is array of bounds and second item is array of values. Bounds and values are + * guaranteed to be consistent. + */ + public synchronized IgniteBiTuple<long[], long[]> histogram() { + long curTs = U.currentTimeMillis(); + + if (curTs >= upperBoundTs) + shiftBuckets(); + + int cnt = (int)((upperBoundTs - lowerBoundTs) / bucketsInterval) + 1; + + long[] res = new long[cnt]; + long[] bounds = new long[cnt]; + + int dummyBucketIdx = dummyBucketIdx(); + + res[0] = outOfBoundsBucket.get() + buckets.get(dummyBucketIdx); + bounds[0] = createTs == lowerBoundTs ? createTs - bucketsInterval : createTs; + + for (int i = 1; i < cnt; i++) { // Starting from 1 (dummyBucketIdx + 1 = index of the first backet). + res[i] = buckets.get((dummyBucketIdx + i) % bucketsCnt); + bounds[i] = lowerBoundTs + (i - 1) * bucketsInterval; + } + + return new IgniteBiTuple<>(bounds, res); + } + + /** + * Gets buckets interval. + */ + public long bucketsInterval() { + return bucketsInterval; + } + + /** + * Gets buckets count. + */ + public int bucketsCount() { + return bucketsCnt; + } + + /** + * Gets start timestamp. + */ + public long startTs() { + return startTs; + } + + /** + * Gets bucket index by timestamp. + * + * Note: Since this method is not synchronized, in case of concurrent reinitialization we can get wrong value here + * without external synchronyzation. + */ + private int bucketIdx(long ts) { + return (int)((ts - startTs) / bucketsInterval) % bucketsCnt; + } + + /** + * Gets index of dummy bucket. + */ + private int dummyBucketIdx() { + return (bucketIdx(lowerBoundTs) + bucketsCnt - 1) % bucketsCnt; + } + + /** + * Change count of items in bucket by given timestamp. + * + * @param ts Timestamp. + * @param val Value to add. + */ + private void add(long ts, int val) { + long curTs = U.currentTimeMillis(); + + assert ts <= curTs : "Unexpected timestamp [curTs = " + curTs + ", ts=" + ts + ']'; + + if (curTs >= upperBoundTs) + shiftBuckets(); + + if (ts < lowerBoundTs) + outOfBoundsBucket.addAndGet(val); + else { + AtomicLongArray buckets = this.buckets; + int idx = bucketIdx(ts); + + if (ts <= startTs) { // Histogram was concurrently reinitialized. + if (ts == startTs) { + synchronized (this) { + // We can't be sure about correct buckets variable without the lock here, but this is the rare + // case and will not affect performance much. + this.buckets.addAndGet(0, val); + } + } + else + outOfBoundsBucket.addAndGet(val); + } + else { + // There is a race between lowerBoundTs check and bucket modification, so we can modify dropped bucket + // in some cases (no more than one bucket behind lowerBoundTs). Dummy bucket was reserved for this + // purpose (to avoid interference of writes to dropped bucket and writes to most recent bucket). + // Values from dummy bucket will be flushed to outOfBoundsBucket during next shift. + buckets.addAndGet(idx, val); + + if (buckets != this.buckets) { + // If histogram was concurrently reinitialized after bucket modification we should save our change + // to not loose it. + outOfBoundsBucket.addAndGet(buckets.getAndSet(idx, 0L)); + } + } + } + } + + /** + * Shift buckets to ensure that upper bound of the buckets array is always greater then current timestamp. + */ + private synchronized void shiftBuckets() { + long curTs = U.currentTimeMillis(); + + long oldLowerBoundTs = lowerBoundTs; + long oldUpperBoundTs = upperBoundTs; + + // Double check under the lock. + if (curTs < oldUpperBoundTs) + return; + + int bucketsSinceLastShift = (int)((curTs - oldUpperBoundTs) / bucketsInterval) + 1; + + long newUpperBoundTs = oldUpperBoundTs + bucketsSinceLastShift * bucketsInterval; + + long newLowerBoundTs = newUpperBoundTs - (bucketsCnt - 1) * bucketsInterval; + + if (newLowerBoundTs > oldLowerBoundTs) { + int bucketsToShift = Math.min(bucketsCnt, (int)((newLowerBoundTs - oldLowerBoundTs) / bucketsInterval)); + + int shiftBucketIdx = (bucketIdx(oldLowerBoundTs) + bucketsCnt - 1) % bucketsCnt; // Start with dummy bucket. + + // Move content of all dropped buckets (including dummy bucket) to the "out of bounds" bucket. + for (int i = 0; i <= bucketsToShift; i++) { + outOfBoundsBucket.addAndGet(buckets.getAndSet(shiftBucketIdx, 0)); + + shiftBucketIdx = (shiftBucketIdx + 1) % bucketsCnt; + } + + lowerBoundTs = newLowerBoundTs; + } + + upperBoundTs = newUpperBoundTs; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/PagesTimestampHistogramView.java b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/PagesTimestampHistogramView.java new file mode 100644 index 00000000000..83c52b918c7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/PagesTimestampHistogramView.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.systemview.view; + +import java.util.Date; +import org.apache.ignite.internal.managers.systemview.walker.Order; + +/** + * Pages timestamp histogramm representation for a {@link SystemView}. + */ +public class PagesTimestampHistogramView { + /** Data region name. */ + private final String dataRegionName; + + /** Start of timestamps interval. */ + private final long intervalStart; + + /** End of timestamps interval. */ + private final long intervalEnd; + + /** Count of pages last accessed within given interval. */ + private final long pagesCnt; + + /** + * @param dataRegionName Data region name. + * @param intervalStart Start of timestamps interval. + * @param intervalEnd End of timestamps interval. + * @param pagesCnt Count of pages last accessed within given interval. + */ + public PagesTimestampHistogramView(String dataRegionName, long intervalStart, long intervalEnd, long pagesCnt) { + this.dataRegionName = dataRegionName; + this.intervalStart = intervalStart; + this.intervalEnd = intervalEnd; + this.pagesCnt = pagesCnt; + } + + /** @return Data region name. */ + @Order + public String dataRegionName() { + return dataRegionName; + } + + /** @return Start of timestamps interval. */ + @Order(1) + public Date intervalStart() { + return new Date(intervalStart); + } + + /** @return End of timestamps interval. */ + @Order(2) + public Date intervalEnd() { + return new Date(intervalEnd); + } + + /** @return Count of pages last accessed within given interval. */ + @Order(3) + public long pagesCount() { + return pagesCnt; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java index 515605b8f9e..353a5e75499 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.metric; +import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -25,6 +26,7 @@ import org.apache.ignite.internal.processors.metric.MetricRegistry; import org.apache.ignite.internal.processors.metric.MetricsMxBeanImpl; import org.apache.ignite.internal.processors.metric.impl.HistogramMetricImpl; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; +import org.apache.ignite.internal.processors.metric.impl.PeriodicHistogramMetricImpl; import org.apache.ignite.mxbean.MetricsMxBean; import org.apache.ignite.spi.metric.HistogramMetric; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -132,6 +134,49 @@ public class MetricsConfigurationTest extends GridCommonAbstractTest { } } + /** Tests configuration of {@link PeriodicHistogramMetricImpl}. */ + @Test + public void testPageTimestampHistogramConfiguration() throws Exception { + String registry = "io.dataregion.default"; + String metricName = "PageTimestampHistogram"; + + IgniteEx g = startGrid("persistent-0"); + + try { + g.cluster().state(ClusterState.ACTIVE); + + MetricsMxBean bean = metricsBean(g); + + PeriodicHistogramMetricImpl histogram = g.context().metric().registry(registry).findMetric(metricName); + + // Check buckets count including dummy bucket. + assertEquals(PeriodicHistogramMetricImpl.DFLT_BUCKETS_CNT + 1, histogram.bucketsCount()); + + // Reconfigure with 5 buckets and 1 minute interval. + long interval = 60_000L; + long[] bounds = new long[] {0L, interval, 0L, 0L, 0L}; + + bean.configureHistogramMetric(metricName(registry, metricName), bounds); + + assertEquals(bounds.length + 1, histogram.bucketsCount()); + assertEquals(interval, histogram.bucketsInterval()); + + // Check configuration after restart. + stopGrid("persistent-0", false); + + g = startGrid("persistent-0"); + g.cluster().state(ClusterState.ACTIVE); + + histogram = g.context().metric().registry(registry).findMetric(metricName); + + assertEquals(bounds.length + 1, histogram.bucketsCount()); + assertEquals(interval, histogram.bucketsInterval()); + } + finally { + g.close(); + } + } + /** Tests metric configuration applied on all nodes. */ @Test public void testConfigurationSeveralNodes() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/PeriodicHistogramMetricImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/PeriodicHistogramMetricImplTest.java new file mode 100644 index 00000000000..8470f7079a4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/PeriodicHistogramMetricImplTest.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metric; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.metric.impl.PeriodicHistogramMetricImpl; +import org.apache.ignite.internal.util.GridTestClockTimer; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test PeriodicHistogramMetricImpl class. + */ +public class PeriodicHistogramMetricImplTest extends GridCommonAbstractTest { + /** Mock for current time */ + private static final AtomicLong curTime = new AtomicLong(System.currentTimeMillis()); + + /** Test time supplier. */ + private static final LongSupplier timeSupplier = curTime::get; + + /** Histogram. */ + PeriodicHistogramMetricImpl histogram; + + /** */ + @BeforeClass + public static void beforeClass() { + GridTestClockTimer.timeSupplier(timeSupplier); + } + + /** */ + @AfterClass + public static void afterClass() { + GridTestClockTimer.timeSupplier(GridTestClockTimer.DFLT_TIME_SUPPLIER); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + histogram = new PeriodicHistogramMetricImpl("test", null); + + super.beforeTest(); + } + + /** */ + @Test + public void testConcurrentUpdate() throws Exception { + long interval = histogram.bucketsInterval(); + int bucketsCnt = histogram.bucketsCount(); + int threadCnt = 20; + int iterations = 1000; + + long startTs = curTime.get(); + + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < iterations; i++) { + long ts = addCurrentTime(interval); + + // Update current data. + histogram.increment(ts); + + // Update historical data around buckets lower bound. + for (int j = bucketsCnt - 10; j < bucketsCnt + 10; j++) { + long ts1 = ts - j * interval; + + if (ts1 <= startTs) + break; + + histogram.increment(ts1); + histogram.decrement(ts1); + } + } + }, threadCnt, "histogram-updater"); + + assertEquals(threadCnt * iterations, Arrays.stream(buckets()).sum()); + } + + /** */ + @Test + public void testConcurrentHistogram() throws Exception { + long interval = histogram.bucketsInterval(); + int bucketsCnt = histogram.bucketsCount(); + int threadCnt = 20; + int valPerBucket = 1000; + int iterations = 1000; + + // Initial fill. + for (int i = 0; i < bucketsCnt; i++) { + long ts = addCurrentTime(interval); + + for (int j = 0; j < valPerBucket; j++) + histogram.increment(ts); + } + + assertEquals(valPerBucket * bucketsCnt, Arrays.stream(buckets()).sum()); + + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < iterations; i++) { + long ts = addCurrentTime(interval); + + for (int j = 0; j < valPerBucket; j++) { + histogram.increment(ts); + histogram.decrement(ts - j * interval); + } + + long sum = Arrays.stream(buckets()).sum(); + + // Check that no buckets were lost during concurrent calculation. + assertTrue("Unexpected items count " + sum, sum >= valPerBucket * bucketsCnt); + } + }, threadCnt, "histogram-updater"); + + assertEquals(valPerBucket * bucketsCnt, Arrays.stream(buckets()).sum()); + } + + /** */ + @Test + public void testConcurrentReinit() throws Exception { + long interval = histogram.bucketsInterval(); + int bucketsCnt = histogram.bucketsCount(); + int threadCnt = 20; + int valPerBucket = 1000; + int iterations = 1000; + + // Initial fill. + for (int i = 0; i < bucketsCnt; i++) { + long ts = addCurrentTime(interval); + + for (int j = 0; j < valPerBucket; j++) + histogram.increment(ts); + } + + assertEquals(valPerBucket * bucketsCnt, Arrays.stream(buckets()).sum()); + + AtomicBoolean finished = new AtomicBoolean(); + + IgniteInternalFuture<?> fut = GridTestUtils.runAsync(() -> { + Random rnd = new Random(); + while (!finished.get()) { + histogram.reinit(interval / 2 + rnd.nextInt((int)interval), + bucketsCnt / 2 + rnd.nextInt(bucketsCnt)); + } + }); + + try { + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < iterations; i++) { + long ts = addCurrentTime(interval); + + for (int j = 0; j < valPerBucket; j++) { + histogram.increment(ts); + histogram.decrement(ts - j * interval); + } + } + }, threadCnt, "histogram-updater"); + } + finally { + finished.set(true); + } + + fut.get(); + + assertEquals(valPerBucket * bucketsCnt, Arrays.stream(buckets()).sum()); + } + + /** */ + @Test + public void testConcurrentLowerBoundBucketUpdate() throws Exception { + long interval = histogram.bucketsInterval(); + int bucketsCnt = histogram.bucketsCount(); + int threadCnt = 20; + int iterations = 1000; + + CyclicBarrier barrier = new CyclicBarrier(threadCnt + 1); + + addCurrentTime(interval * bucketsCnt); + + IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(() -> { + try { + for (int i = 0; i < iterations; i++) { + barrier.await(1, TimeUnit.SECONDS); + + long ts = curTime.get() - interval * (bucketsCnt - 2); // 1 dummy bucket + 1 shifted bucket. + + barrier.await(1, TimeUnit.SECONDS); + + histogram.increment(ts); + + // Maximize probability of collision between buckets shift and buckets update. + for (int j = 0; j < 10; j++) { + histogram.decrement(ts); + histogram.increment(ts); + } + } + } + catch (Exception e) { + throw new IgniteException(e); + } + }, threadCnt, "histogram-updater"); + + for (int i = 0; i < iterations; i++) { + barrier.await(1, TimeUnit.SECONDS); + + assertEquals(i * threadCnt, Arrays.stream(buckets()).sum()); + assertEquals(i * threadCnt, bucket(0)); + + barrier.await(1, TimeUnit.SECONDS); + + histogram.increment(curTime.get()); + histogram.decrement(curTime.get()); + + addCurrentTime(interval); + + long[] hist = buckets(); + + assertTrue("Unexpected items count " + hist[0] + ", expected between " + (i * threadCnt) + " and " + + (i + 1) * threadCnt, hist[0] >= i * threadCnt && hist[0] <= (i + 1) * threadCnt); + + for (int j = 1; j < hist.length; j++) + assertEquals(0, hist[j]); + } + + fut.get(); + } + + /** */ + @Test + public void testShiftOneBucket() { + long interval = histogram.bucketsInterval(); + + long ts = histogram.startTs(); + + addCurrentTime(ts - curTime.get() + interval - 1); + + histogram.increment(ts); + histogram.increment(ts + 1); + histogram.increment(ts + interval - 1); + + assertEquals(3, bucket(-1)); + + addCurrentTime(1); + + assertEquals(0, bucket(-1)); + assertEquals(3, bucket(-2)); + + ts = curTime.get(); + + addCurrentTime(interval - 1); + + histogram.increment(ts); + histogram.increment(ts + 1); + histogram.increment(ts + interval / 2); + histogram.increment(ts + interval - 1); + + assertEquals(4, bucket(-1)); + assertEquals(3, bucket(-2)); + + addCurrentTime(1); + + assertEquals(0, bucket(-1)); + assertEquals(4, bucket(-2)); + assertEquals(3, bucket(-3)); + } + + /** */ + @Test + public void testShiftMoreThanOneBucket() { + long interval = histogram.bucketsInterval(); + int bucketsCnt = histogram.bucketsCount(); + + long ts = curTime.get(); + + histogram.increment(ts); + + assertEquals(1, bucket(-1)); + + ts = addCurrentTime(interval); + + histogram.increment(ts); + + assertEquals(1, bucket(-1)); + assertEquals(1, bucket(-2)); + + ts = addCurrentTime(interval * (bucketsCnt - 2)); + + histogram.increment(ts); + + assertEquals(1, bucket(-1)); + assertEquals(1, bucket(1)); + assertEquals(1, bucket(0)); + + for (int i = -1; i <= 1; i++) { + ts = addCurrentTime(interval * (bucketsCnt + i)); + + histogram.increment(ts); + + assertEquals(1, bucket(-1)); + assertEquals(4 + i, bucket(0)); + } + + addCurrentTime(interval * bucketsCnt); + + // Check shift without modification. + assertEquals(6, bucket(0)); + } + + /** + * Gets bucket values of current histogram. + */ + private long[] buckets() { + return histogram.histogram().get2(); + } + + /** + * @param idx Bucket index (if < 0 - index from the end). + */ + private long bucket(int idx) { + long[] buckets = buckets(); + + return idx >= 0 ? buckets[idx] : buckets[buckets.length + idx]; + } + + /** + * Add value to U.currentTimeMillis(). + * + * @param time Time. + */ + private static long addCurrentTime(long time) { + long res = curTime.addAndGet(time); + + GridTestClockTimer.update(); + + return res; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java index 5b367cf6aa3..d391435979f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewSelfTest.java @@ -34,9 +34,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import javax.cache.Cache; - import com.google.common.collect.Lists; import org.apache.ignite.IgniteAtomicLong; import org.apache.ignite.IgniteAtomicReference; @@ -45,6 +45,7 @@ import org.apache.ignite.IgniteAtomicStamped; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCompute; import org.apache.ignite.IgniteCountDownLatch; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteJdbcThinDriver; import org.apache.ignite.IgniteLock; @@ -82,8 +83,10 @@ import org.apache.ignite.internal.managers.systemview.walker.NodeAttributeViewWa import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorage; +import org.apache.ignite.internal.processors.metric.impl.PeriodicHistogramMetricImpl; import org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext; import org.apache.ignite.internal.processors.service.DummyService; +import org.apache.ignite.internal.util.GridTestClockTimer; import org.apache.ignite.internal.util.StripedExecutor; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -110,6 +113,7 @@ import org.apache.ignite.spi.systemview.view.MetastorageView; import org.apache.ignite.spi.systemview.view.NodeAttributeView; import org.apache.ignite.spi.systemview.view.NodeMetricsView; import org.apache.ignite.spi.systemview.view.PagesListView; +import org.apache.ignite.spi.systemview.view.PagesTimestampHistogramView; import org.apache.ignite.spi.systemview.view.ScanQueryView; import org.apache.ignite.spi.systemview.view.ServiceView; import org.apache.ignite.spi.systemview.view.SnapshotView; @@ -146,6 +150,7 @@ import static org.apache.ignite.internal.processors.cache.GridCacheUtils.cacheId import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.BINARY_METADATA_VIEW; import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.METASTORE_VIEW; import static org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager.DATA_REGION_PAGE_LIST_VIEW; +import static org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager.PAGE_TS_HISTOGRAM_VIEW; import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME; import static org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager.TXS_MON_LIST; import static org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor.BASELINE_NODES_SYS_VIEW; @@ -189,6 +194,20 @@ public class SystemViewSelfTest extends GridCommonAbstractTest { /** */ public static final String TEST_TRANSFORMER = "TestTransformer"; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + cleanPersistenceDir(); + } + /** Tests work of {@link SystemView} for caches. */ @Test public void testCachesView() throws Exception { @@ -2104,6 +2123,194 @@ public class SystemViewSelfTest extends GridCommonAbstractTest { } } + /** */ + @Test + public void testPagesTimestampHistogram() throws Exception { + int keysCnt = 50_000; + + AtomicLong curTime = new AtomicLong(System.currentTimeMillis()); + + GridTestClockTimer.timeSupplier(curTime::get); + GridTestClockTimer.update(); + + String regionName = "default"; + + DataStorageConfiguration dsCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setMaxSize(50L * 1024 * 1024) + .setPersistenceEnabled(true) + .setName(regionName) + .setMetricsEnabled(true) + ); + + cleanPersistenceDir(); + + try (IgniteEx ignite = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg))) { + ignite.cluster().state(ClusterState.ACTIVE); + + CacheConfiguration<Object, Object> ccfg1 = new CacheConfiguration<>("test-pages-ts1") + .setAffinity(new RendezvousAffinityFunction(false, 10)); + + CacheConfiguration<Object, Object> ccfg2 = new CacheConfiguration<>("test-pages-ts2") + .setAffinity(new RendezvousAffinityFunction(false, 10)); + + IgniteCache<Object, Object> cache1 = ignite.createCache(ccfg1); + + long ts1 = curTime.get(); + + for (int i = 0; i < 1000; i++) + cache1.put(i, i); + + long ts2 = curTime.addAndGet(PeriodicHistogramMetricImpl.DFLT_BUCKETS_INTERVAL); + GridTestClockTimer.update(); + + for (int i = 1000; i < 2000; i++) + cache1.put(i, i); + + SystemView<PagesTimestampHistogramView> pagesTsHistogram = + ignite.context().systemView().view(PAGE_TS_HISTOGRAM_VIEW); + + assertNotNull(pagesTsHistogram); + + long totalCnt = 0; + + for (PagesTimestampHistogramView view : pagesTsHistogram) { + if (regionName.equals(view.dataRegionName())) { + if ((ts1 >= view.intervalStart().getTime() && ts1 <= view.intervalEnd().getTime()) || + (ts2 >= view.intervalStart().getTime() && ts2 <= view.intervalEnd().getTime())) { + assertTrue("Unexpected pages count: " + view.pagesCount(), view.pagesCount() > 0); + + totalCnt += view.pagesCount(); + } + else + assertEquals(0, view.pagesCount()); + } + } + + assertTrue(totalCnt > 0); + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), totalCnt); + + assertEquals(2, F.size(F.iterator(pagesTsHistogram, v -> v, true, v -> v.pagesCount() > 0))); + + // Check histogram after replacement. + long ts3 = curTime.addAndGet(PeriodicHistogramMetricImpl.DFLT_BUCKETS_INTERVAL); + GridTestClockTimer.update(); + + ignite.createCache(ccfg2); + + try (IgniteDataStreamer<Integer, Object> streamer = ignite.dataStreamer("test-pages-ts2")) { + for (int i = 0; i < keysCnt; i++) + streamer.addData(i, new byte[1000]); + } + + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), + F.sumInt(F.iterator(pagesTsHistogram, v -> (int)v.pagesCount(), true))); + + assertFalse(F.isEmpty(F.iterator(pagesTsHistogram, v -> v, true, v -> v.pagesCount() > 0 && + v.intervalStart().getTime() <= ts3 && ts3 <= v.intervalEnd().getTime()))); + + // Check histogram after cache destroy and remove of outdated pages. + long ts4 = curTime.addAndGet(PeriodicHistogramMetricImpl.DFLT_BUCKETS_INTERVAL); + GridTestClockTimer.update(); + + ignite.destroyCache("test-pages-ts2"); + + IgniteCache<Object, Object> cache2 = ignite.createCache(ccfg2); + + for (int i = 0; i < keysCnt; i++) { + cache1.put(i, i); + cache2.put(i, new byte[1000]); + } + + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), + F.sumInt(F.iterator(pagesTsHistogram, v -> (int)v.pagesCount(), true))); + + assertFalse(F.isEmpty(F.iterator(pagesTsHistogram, v -> v, true, v -> v.pagesCount() > 0 && + v.intervalStart().getTime() <= ts4 && ts4 <= v.intervalEnd().getTime()))); + } + finally { + GridTestClockTimer.timeSupplier(GridTestClockTimer.DFLT_TIME_SUPPLIER); + } + } + + /** */ + @Test + public void testPagesTimestampHistogramAfterPartitionEviction() throws Exception { + int keysCnt = 50_000; + + String regionName = "default"; + + DataStorageConfiguration dsCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setMaxSize(50L * 1024 * 1024) + .setPersistenceEnabled(true) + .setName(regionName) + .setMetricsEnabled(true) + ); + + cleanPersistenceDir(); + + try (IgniteEx ignite = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg))) { + ignite.cluster().state(ClusterState.ACTIVE); + + IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>("test-pages-ts") + .setBackups(1).setAffinity(new RendezvousAffinityFunction(false, 10))); + + try (IgniteDataStreamer<Integer, Object> streamer = ignite.dataStreamer("test-pages-ts")) { + for (int i = 0; i < keysCnt; i++) + streamer.addData(i, new byte[1000]); + } + + startGrid(getConfiguration(getTestIgniteInstanceName(1)).setDataStorageConfiguration(dsCfg)); + startGrid(getConfiguration(getTestIgniteInstanceName(2)).setDataStorageConfiguration(dsCfg)); + + resetBaselineTopology(); + + awaitPartitionMapExchange(true, true, null); + + // Force checkpoint to invalidate evicted partitions. + forceCheckpoint(ignite); + + // Check histogram after partition eviction. + SystemView<PagesTimestampHistogramView> pagesTsHistogram = + ignite.context().systemView().view(PAGE_TS_HISTOGRAM_VIEW); + + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), + F.sumInt(F.iterator(pagesTsHistogram, v -> (int)v.pagesCount(), true))); + + stopGrid(2); + + resetBaselineTopology(); + + // Wait until rebalance complete. + assertTrue(GridTestUtils.waitForCondition(() -> ignite.context().discovery().topologyVersionEx() + .minorTopologyVersion() >= 2, 5_000L)); + + // Check histogram after rebalance. + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), + F.sumInt(F.iterator(pagesTsHistogram, v -> (int)v.pagesCount(), true))); + + stopGrid(1); + + resetBaselineTopology(); + + // Allocate some pages after eviction. + for (int i = 0; i < 10_000; i++) + cache.put(i + keysCnt, new byte[1024]); + + // Acquire some outdated pages. + for (int i = 0; i < keysCnt + 10_000; i++) + assertNotNull(cache.get(i)); + + // Check histogram after replacement of outdated pages. + assertEquals(ignite.dataRegionMetrics(regionName).getPhysicalMemoryPages(), + F.sumInt(F.iterator(pagesTsHistogram, v -> (int)v.pagesCount(), true))); + } + finally { + stopAllGrids(); + } + } + /** Test node filter. */ public static class TestNodeFilter implements IgnitePredicate<ClusterNode> { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java index b6822938704..caa74caf74c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java @@ -688,6 +688,7 @@ public class PageMemoryImplTest extends GridCommonAbstractTest { } }; + mem.metrics().pageMemory(mem); mem.metrics().enableMetrics(); mem.start(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java index b058c364f32..eba1e484200 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java @@ -17,10 +17,21 @@ package org.apache.ignite.internal.util; +import java.util.function.LongSupplier; + /** * Clock timer for tests. */ public class GridTestClockTimer implements Runnable { + /** Default time supplier. */ + public static final LongSupplier DFLT_TIME_SUPPLIER = System::currentTimeMillis; + + /** Current time supplier. */ + private static volatile LongSupplier timeSupplier = DFLT_TIME_SUPPLIER; + + /** Mutex to avoid races between time updates. */ + private static final Object mux = new Object(); + /** * Constructor. */ @@ -41,10 +52,28 @@ public class GridTestClockTimer implements Runnable { } } + /** + * Sets new time supplier. + * + * @param timeSupplier Time supplier. + */ + public static void timeSupplier(LongSupplier timeSupplier) { + GridTestClockTimer.timeSupplier = timeSupplier; + } + + /** + * Updates current time with value supplied by time supplier. + */ + public static void update() { + synchronized (mux) { + IgniteUtils.curTimeMillis = timeSupplier.getAsLong(); + } + } + /** {@inheritDoc} */ @Override public void run() { while (true) { - IgniteUtils.curTimeMillis = System.currentTimeMillis(); + update(); try { Thread.sleep(10); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java index ffdf0ce643c..e2419dc0839 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java @@ -39,6 +39,7 @@ import org.apache.ignite.internal.managers.IgniteDiagnosticPartitionReleaseFutur import org.apache.ignite.internal.managers.communication.GridIoManagerFileTransmissionSelfTest; import org.apache.ignite.internal.managers.discovery.IncompleteDeserializationExceptionTest; import org.apache.ignite.internal.metric.MetricsClusterActivationTest; +import org.apache.ignite.internal.metric.PeriodicHistogramMetricImplTest; import org.apache.ignite.internal.mxbean.IgniteStandardMXBeanTest; import org.apache.ignite.internal.pagemem.wal.record.WALRecordSerializationTest; import org.apache.ignite.internal.pagemem.wal.record.WALRecordTest; @@ -224,7 +225,8 @@ import org.junit.runners.Suite; ExponentialBackoffTest.class, ProgressSpeedCalculationTest.class, - ConcurrentMappingFileReadWriteTest.class + ConcurrentMappingFileReadWriteTest.class, + PeriodicHistogramMetricImplTest.class, }) public class IgniteBasicTestSuite2 { } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java index 0500db1af26..d91d444b0e5 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java @@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.metric; import java.lang.reflect.Field; import java.sql.Connection; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -72,6 +73,7 @@ import org.apache.ignite.internal.processors.service.DummyService; import org.apache.ignite.internal.util.StripedExecutor; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.services.ServiceConfiguration; import org.apache.ignite.spi.systemview.view.MetastorageView; @@ -115,10 +117,15 @@ public class SqlViewExporterSpiTest extends AbstractExporterSpiTest { cfg.setDataStorageConfiguration(new DataStorageConfiguration() .setDataRegionConfigurations( - new DataRegionConfiguration().setName("in-memory").setMaxSize(100L * 1024 * 1024)) + new DataRegionConfiguration() + .setName("in-memory") + .setMaxSize(100L * 1024 * 1024) + .setMetricsEnabled(true)) .setDefaultDataRegionConfiguration( new DataRegionConfiguration() - .setPersistenceEnabled(true))); + .setName("persistent") + .setPersistenceEnabled(true) + .setMetricsEnabled(true))); return cfg; } @@ -162,7 +169,7 @@ public class SqlViewExporterSpiTest extends AbstractExporterSpiTest { @Test public void testDataRegionMetrics() throws Exception { List<List<?>> res = execute(ignite0, - "SELECT REPLACE(name, 'io.dataregion.default.'), value, description FROM SYS.METRICS"); + "SELECT REPLACE(name, 'io.dataregion.persistent.'), value, description FROM SYS.METRICS"); Set<String> names = new HashSet<>(); @@ -446,7 +453,8 @@ public class SqlViewExporterSpiTest extends AbstractExporterSpiTest { "DS_REENTRANTLOCKS", "DS_SETS", "DS_SEMAPHORES", - "DS_QUEUES" + "DS_QUEUES", + "PAGES_TIMESTAMP_HISTOGRAM" )); Set<String> actViews = new TreeSet<>(); @@ -1190,6 +1198,32 @@ public class SqlViewExporterSpiTest extends AbstractExporterSpiTest { "SELECT * FROM SYS.SNAPSHOT WHERE cache_groups LIKE '%" + DEFAULT_CACHE_NAME + "%'").size()); } + /** */ + @Test + public void testPagesTimestampHistogram() throws Exception { + IgniteCache<Integer, Integer> cache = ignite0.getOrCreateCache("test-page-ts-cache"); + + cache.put(0, 0); + + assertEquals(0, execute(ignite0, + "SELECT * FROM SYS.PAGES_TIMESTAMP_HISTOGRAM WHERE DATA_REGION_NAME = ?", "in-memory").size()); + + // There should be two buckets after start: empty "out of bounds" bucket and current bucket. + assertEquals(2, execute(ignite0, + "SELECT * FROM SYS.PAGES_TIMESTAMP_HISTOGRAM WHERE DATA_REGION_NAME = ?", "persistent").size()); + + Timestamp ts = new Timestamp(U.currentTimeMillis()); + + List<List<?>> res = execute(ignite0, "SELECT INTERVAL_START, INTERVAL_END " + + "FROM SYS.PAGES_TIMESTAMP_HISTOGRAM " + + "WHERE DATA_REGION_NAME = ? AND PAGES_COUNT > 0", "persistent"); + + assertEquals(1, res.size()); + + assertTrue(ts.compareTo(((Timestamp)res.get(0).get(0))) >= 0); + assertTrue(ts.compareTo(((Timestamp)res.get(0).get(1))) <= 0); + } + /** * Execute query on given node. *