IGNITE-5024 default Memory Policy size was changed to 80% of available RAM
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83d913c8 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83d913c8 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83d913c8 Branch: refs/heads/ignite-5024 Commit: 83d913c804ab6ecd4fb650e3b09062dca92ccae1 Parents: 9ce62e6 Author: Sergey Chugunov <[email protected]> Authored: Wed Apr 19 17:03:32 2017 +0300 Committer: Sergey Chugunov <[email protected]> Committed: Thu Apr 20 12:25:39 2017 +0300 ---------------------------------------------------------------------- .../configuration/MemoryConfiguration.java | 40 +++++++++- .../IgniteCacheDatabaseSharedManager.java | 33 ++++++++- .../ignite/internal/util/IgniteUtils.java | 26 ++++++- .../cache/MemoryPolicyConfigValidationTest.java | 77 +++++++++++++++++++- 4 files changed, 162 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/83d913c8/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java index de8fdbf..3cccb42 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java @@ -62,8 +62,11 @@ public class MemoryConfiguration implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** Default memory policy's size (1 GB). */ - public static final long DFLT_MEMORY_POLICY_SIZE = 1024 * 1024 * 1024; + /** Fraction of available memory to allocate for default MemoryPolicy. */ + private static final double DFLT_MEMORY_POLICY_FRACTION = 0.8; + + /** Default memory policy's size is 80% of physical memory available on current machine. */ + public static final long DFLT_MEMORY_POLICY_SIZE = (long) DFLT_MEMORY_POLICY_FRACTION * U.getTotalMemoryAvailable(); /** Default size of a memory chunk for the system cache (100 MB). */ public static final long DFLT_SYS_CACHE_MEM_SIZE = 100 * 1024 * 1024; @@ -83,6 +86,9 @@ public class MemoryConfiguration implements Serializable { /** A name of the memory policy that defines the default memory region. */ private String dfltMemPlcName; + /** Size of memory (in bytes) to use for default MemoryPolicy. */ + private Long dfltMemPlcSize; + /** Memory policies. */ private MemoryPolicyConfiguration[] memPlcs; @@ -164,8 +170,10 @@ public class MemoryConfiguration implements Serializable { public MemoryPolicyConfiguration createDefaultPolicyConfig() { MemoryPolicyConfiguration memPlc = new MemoryPolicyConfiguration(); + long size = (dfltMemPlcSize != null) ? dfltMemPlcSize : DFLT_MEMORY_POLICY_SIZE; + memPlc.setName(null); - memPlc.setSize(DFLT_MEMORY_POLICY_SIZE); + memPlc.setSize(size); return memPlc; } @@ -189,6 +197,32 @@ public class MemoryConfiguration implements Serializable { } /** + * Gets a size for default memory policy overridden by user. + * + * @return default memory policy size overridden by user or -1 if nothing was specified. + */ + public long getDefaultMemoryPolicySize() { + return (dfltMemPlcSize != null) ? dfltMemPlcSize : -1; + } + + /** + * Overrides size of default memory policy which is created automatically. + * + * If user doesn't specify any memory policy configuration, a default one with default size + * (80% of available RAM) is created by Ignite. + * + * This property allows user to specify desired size of default memory policy + * without having to use more verbose syntax of MemoryPolicyConfiguration elements. + * + * @param dfltMemPlcSize Size of default memory policy overridden by user. + */ + public MemoryConfiguration setDefaultMemoryPolicySize(long dfltMemPlcSize) { + this.dfltMemPlcSize = dfltMemPlcSize; + + return this; + } + + /** * Gets a name of default memory policy. * * @return A name of a custom memory policy configured with {@link MemoryConfiguration} or {@code null} of the http://git-wip-us.apache.org/repos/asf/ignite/blob/83d913c8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java index a34aed2..de882da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java @@ -296,20 +296,40 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap } } - checkDefaultPolicyConfiguration(dbCfg.getDefaultMemoryPolicyName(), plcNames); + checkDefaultPolicyConfiguration( + dbCfg.getDefaultMemoryPolicyName(), + dbCfg.getDefaultMemoryPolicySize(), + plcNames); } /** * @param dfltPlcName Default MemoryPolicy name. + * @param dfltPlcSize Default size of MemoryPolicy overridden by user (equals to -1 if wasn't specified by user). * @param plcNames All MemoryPolicy names. * @throws IgniteCheckedException In case of validation violation. */ - private static void checkDefaultPolicyConfiguration(String dfltPlcName, Set<String> plcNames) throws IgniteCheckedException { + private static void checkDefaultPolicyConfiguration(String dfltPlcName, + long dfltPlcSize, + Set<String> plcNames) throws IgniteCheckedException { + if (dfltPlcSize != -1) { + if (dfltPlcName != null) + throw new IgniteCheckedException("User-defined MemoryPolicy configuration " + + "and defaultMemoryPolicySize properties are set at the same time. " + + "Delete either MemoryConfiguration.defaultMemoryPolicySize property " + + "or configuration of user-defined default MemoryPolicy"); + + + if (dfltPlcSize < MIN_PAGE_MEMORY_SIZE) + throw new IgniteCheckedException("User-defined default MemoryPolicy size is less than 1MB. " + + "Use MemoryConfiguration.defaultMemoryPolicySize property to set correct size."); + } + if (dfltPlcName != null) { if (dfltPlcName.isEmpty()) throw new IgniteCheckedException("User-defined default MemoryPolicy name must be non-empty"); if (!plcNames.contains(dfltPlcName)) - throw new IgniteCheckedException("User-defined default MemoryPolicy name must be presented among configured MemoryPolices: " + dfltPlcName); + throw new IgniteCheckedException("User-defined default MemoryPolicy name " + + "must be presented among configured MemoryPolices: " + dfltPlcName); } } @@ -319,7 +339,12 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap */ private static void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException { if (plcCfg.getSize() < MIN_PAGE_MEMORY_SIZE) - throw new IgniteCheckedException("MemoryPolicy must have size more than 1MB: " + plcCfg.getName()); + throw new IgniteCheckedException("MemoryPolicy must have size more than 1MB: MemoryPolicy[name=" + + plcCfg.getName() + + "; size=" + + plcCfg.getSize() + + "]. Use MemoryPolicyConfiguration.size property to set correct size (in bytes)." + ); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/83d913c8/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 7d7d071..fde8cbf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -177,8 +177,6 @@ import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.IgniteNodeAttributes; -import org.apache.ignite.internal.binary.BinaryObjectEx; -import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException; @@ -5629,6 +5627,28 @@ public abstract class IgniteUtils { } /** + * Gets amount of RAM memory available on this machine. + * + * @return Total amount of memory in bytes or -1 if any exception happened. + */ + public static long getTotalMemoryAvailable() { + MBeanServer mBeanSrv = ManagementFactory.getPlatformMBeanServer(); + + Object attr; + + try { + attr = mBeanSrv.getAttribute( + ObjectName.getInstance("java.lang", "type", "OperatingSystem"), + "TotalPhysicalMemorySize"); + } + catch (Exception e) { + return -1; + } + + return (attr instanceof Long) ? (Long) attr : -1; + } + + /** * Gets compilation MBean. * * @return Compilation MBean. @@ -9554,7 +9574,7 @@ public abstract class IgniteUtils { public static <T extends R, R> List<R> arrayList(Collection<T> c, @Nullable IgnitePredicate<? super T>... p) { assert c != null; - return IgniteUtils.arrayList(c, c.size(), p); + return arrayList(c, c.size(), p); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/83d913c8/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MemoryPolicyConfigValidationTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MemoryPolicyConfigValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MemoryPolicyConfigValidationTest.java index c0f74d0..565ba99 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MemoryPolicyConfigValidationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MemoryPolicyConfigValidationTest.java @@ -26,6 +26,15 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; * */ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { + /** */ + private static final String VALID_DEFAULT_MEM_PLC_NAME = "valid_dlft_mem_plc"; + + /** */ + private static final String VALID_USER_MEM_PLC_NAME = "valid_user_mem_plc"; + + /** */ + private static final String MISSING_DEFAULT_MEM_PLC_NAME = "missing_mem_plc"; + /** Configuration violation type to check. */ private ValidationViolationType violationType; @@ -61,9 +70,25 @@ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { case MISSING_USER_DEFINED_DEFAULT: plcs = createMissingUserDefinedDefault(); - memCfg.setDefaultMemoryPolicyName("missingMemoryPolicyName"); + memCfg.setDefaultMemoryPolicyName(MISSING_DEFAULT_MEM_PLC_NAME); + + break; + + case TOO_SMALL_USER_DEFINED_DFLT_MEM_PLC_SIZE: + memCfg.setDefaultMemoryPolicySize(1); break; + + case DEFAULT_SIZE_IS_DEFINED_TWICE: + plcs = createValidUserDefault(); + + memCfg.setDefaultMemoryPolicyName(VALID_DEFAULT_MEM_PLC_NAME); + memCfg.setDefaultMemoryPolicySize(10 * 1014 * 1024); + + break; + + default: + fail("Violation type was not configured: " + violationType); } memCfg.setMemoryPolicies(plcs); @@ -73,14 +98,31 @@ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { return cfg; } + /** + * + */ + private MemoryPolicyConfiguration[] createValidUserDefault() { + MemoryPolicyConfiguration[] res = new MemoryPolicyConfiguration[1]; + + res[0] = createMemoryPolicy(VALID_DEFAULT_MEM_PLC_NAME, 100 * 1024 * 1024); + + return res; + } + + /** + * + */ private MemoryPolicyConfiguration[] createMissingUserDefinedDefault() { MemoryPolicyConfiguration[] res = new MemoryPolicyConfiguration[1]; - res[0] = createMemoryPolicy("presentedPolicyCfg", 10 * 1024 * 1024); + res[0] = createMemoryPolicy(VALID_USER_MEM_PLC_NAME, 10 * 1024 * 1024); return res; } + /** + * + */ private MemoryPolicyConfiguration[] createPlcWithNullName() { MemoryPolicyConfiguration[] res = new MemoryPolicyConfiguration[1]; @@ -95,7 +137,7 @@ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { private MemoryPolicyConfiguration[] createTooSmallMemoryCfg() { MemoryPolicyConfiguration[] res = new MemoryPolicyConfiguration[1]; - res[0] = createMemoryPolicy("dflt", 10); + res[0] = createMemoryPolicy(VALID_DEFAULT_MEM_PLC_NAME, 10); return res; } @@ -189,6 +231,25 @@ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { } /** + * User-defined size of default MemoryPolicy must be at least 1MB. + */ + public void testUserDefinedDefaultMemoryTooSmall() throws Exception { + violationType = ValidationViolationType.TOO_SMALL_USER_DEFINED_DFLT_MEM_PLC_SIZE; + + doTest(violationType); + } + + /** + * Defining size of default MemoryPolicy twice with and through <b>defaultMemoryPolicySize</b> property + * and using <b>MemoryPolicyConfiguration</b> description is prohibited. + */ + public void testDefaultMemoryPolicySizeDefinedTwice() throws Exception { + violationType = ValidationViolationType.DEFAULT_SIZE_IS_DEFINED_TWICE; + + doTest(violationType); + } + + /** * Tries to start ignite node with invalid configuration and checks that corresponding exception is thrown. * * @param violationType Configuration violation type. @@ -226,7 +287,15 @@ public class MemoryPolicyConfigValidationTest extends GridCommonAbstractTest { NULL_NAME_ON_USER_DEFINED_POLICY("User-defined MemoryPolicyConfiguration must have non-null and non-empty name."), /** */ - MISSING_USER_DEFINED_DEFAULT("User-defined default MemoryPolicy name must be presented among configured MemoryPolices: "); + MISSING_USER_DEFINED_DEFAULT("User-defined default MemoryPolicy name must be presented among configured MemoryPolices: "), + + /** */ + DEFAULT_SIZE_IS_DEFINED_TWICE("User-defined MemoryPolicy configuration and defaultMemoryPolicySize properties are set at the same time."), + + /** */ + TOO_SMALL_USER_DEFINED_DFLT_MEM_PLC_SIZE("User-defined default MemoryPolicy size is less than 1MB."); + + /** * @param violationMsg Violation message.
