Memory usage metric doesn't come down when memory is freed up

[colinc]

I am using the Ignite metrics API to monitor memory usage levels in an
attempt to avoid OOM conditions.

I have found that the metrics API appears to provide reasonably accurate
figures as entries are being written to the cache - but usage levels do not
come down again when entries are removed. I have tried removing all entries,
individual entries and even destroying the cache. I have also tried waiting
for a significant period of time.

Even when the cache is destroyed, the memory usage figure does not appear to
drop. In fact, removing entries can even cause an increase in the figure.
Despite the metrics, it is possible to insert new entries following removal
of the old ones - indicating that space is in fact available.

A reproducer is below and produces results like:

Out of memory: CachePartialUpdateException after 89MB
Memory used: 99.39624%
Memory used: 99.41027%
Memory used: 99.41406%



package mytest;

import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.failure.NoOpFailureHandler;
import org.junit.Test;

import javax.cache.CacheException;

public class MemoryTest {

    private static final String CACHE_NAME = "cache";
    private static final String DEFAULT_MEMORY_REGION = "Default_Region";
    private static final long MEM_SIZE = 100L * 1024 * 1024;


    @Test
    public void testOOM() throws InterruptedException {
        try (Ignite ignite = startIgnite("IgniteMemoryMonitorTest1")) {
            fillDataRegion(ignite);
            IgniteCache<Object, Object> cache =
                    ignite.getOrCreateCache(CACHE_NAME);

            // Clear all entries from the cache to free up memory 
            memUsed(ignite);
            cache.clear();      // Fails here
            cache.put("Key", "Value");
            memUsed(ignite);

            cache.destroy();
            Thread.sleep(5000);
            memUsed(ignite);
        }
    }


    private Ignite startIgnite(String instanceName) {
        IgniteConfiguration cfg = new IgniteConfiguration();
        cfg.setIgniteInstanceName(instanceName);
        cfg.setDataStorageConfiguration(createDataStorageConfiguration());
        cfg.setFailureHandler(new NoOpFailureHandler());
        return Ignition.start(cfg);
    }

    private DataStorageConfiguration createDataStorageConfiguration() {
        return new DataStorageConfiguration()
                .setDefaultDataRegionConfiguration(
                        new DataRegionConfiguration()
                                .setName(DEFAULT_MEMORY_REGION)
                                .setInitialSize(MEM_SIZE)
                                .setMaxSize(MEM_SIZE)
                                .setMetricsEnabled(true));
    }


    private void fillDataRegion(Ignite ignite) {
        byte[] megabyte = new byte[1024 * 1024];

        int storedDataMB = 0;
        try {
            IgniteCache<Object, Object> cache =
                    ignite.getOrCreateCache(CACHE_NAME);
            for (int i = 0; i < 200; i++) {
                cache.put(i, megabyte);
                storedDataMB++;

                memUsed(ignite);
            }
        } catch (CacheException e) {
            System.out.println("Out of memory: " +
e.getClass().getSimpleName() + " after " + storedDataMB + "MB");
        }
    }

    private void memUsed(Ignite ignite) {
        DataRegionConfiguration defaultDataRegionCfg =
ignite.configuration()
                .getDataStorageConfiguration()
                .getDefaultDataRegionConfiguration();
        String regionName = defaultDataRegionCfg.getName();
        DataRegionMetrics metrics = ignite.dataRegionMetrics(regionName);
        float usedMem = metrics.getPagesFillFactor() *
metrics.getTotalAllocatedPages() * metrics.getPageSize();
        float pctUsed = 100 * usedMem / defaultDataRegionCfg.getMaxSize();
        System.out.println("Memory used: " + pctUsed + "%");
    }
} 



--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/