[ignite] branch master updated: IGNITE-10913 Reduce heap occupation by o.a.i.i.processors.cache.persistence.file.FilePageStore instances - Fixes #6500.
This is an automated email from the ASF dual-hosted git repository. ipavlukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new de16d6b IGNITE-10913 Reduce heap occupation by o.a.i.i.processors.cache.persistence.file.FilePageStore instances - Fixes #6500. de16d6b is described below commit de16d6b47c9d3ccd6b172a27540a6a7d2d0a9b5b Author: denis-chudov AuthorDate: Thu Jun 13 19:32:24 2019 +0300 IGNITE-10913 Reduce heap occupation by o.a.i.i.processors.cache.persistence.file.FilePageStore instances - Fixes #6500. Signed-off-by: ipavlukhin --- .../jol/FileStoreHeapUtilizationJolBenchmark.java | 194 + .../cache/persistence/file/FilePageStore.java | 100 --- .../persistence/file/FilePageStoreFactory.java | 24 ++- .../persistence/file/FilePageStoreManager.java | 15 +- .../cache/persistence/file/FilePageStoreV2.java| 13 +- .../file/FileVersionCheckingFactory.java | 35 ++-- .../persistence/IgnitePdsTaskCancelingTest.java| 2 +- 7 files changed, 327 insertions(+), 56 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java new file mode 100644 index 000..82dd5ca --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.benchmarks.jol; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.openjdk.jol.info.GraphLayout; + +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; + +/** + * + */ +public class FileStoreHeapUtilizationJolBenchmark { +/** */ +private static final String CACHE_NAME = "testCache"; + +/** */ +private static final String HEAP_USAGE = "heap usage"; + +/** */ +private static final String CACHE_WORK_TIME = "cache work time"; + +/** */ +private static final TestResultParameterInfo HEAP_USAGE_PARAM = new TestResultParameterInfo(HEAP_USAGE, false); + +/** */ +private static final TestResultParameterInfo CACHE_WORK_TIME_PARAM = new TestResultParameterInfo(CACHE_WORK_TIME, false); + +/** + * Cleans persistent directory. + * + * @throws Exception if failed. + */ +private void cleanPersistenceDir() throws Exception { +if (!F.isEmpty(G.allGrids())) +throw new IgniteException("Grids are not stopped"); + +U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false)); +U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false)); +U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false)); +U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false)); +} + +/** */ +private IgniteConfiguration getConfiguration(String igniteInstanceName) { +IgniteConfiguration cfg = new IgniteConfiguration(); + +
[ignite] branch master updated: IGNITE-9409 Use current version, modern URL for Ignite download - Fixes #6599.
This is an automated email from the ASF dual-hosted git repository. ilyak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new ebb3659 IGNITE-9409 Use current version, modern URL for Ignite download - Fixes #6599. ebb3659 is described below commit ebb3659f4346b4d86c1460b764a030dd54d2b394 Author: Ilya Kasnacheev AuthorDate: Thu Jun 13 18:18:10 2019 +0300 IGNITE-9409 Use current version, modern URL for Ignite download - Fixes #6599. Signed-off-by: Ilya Kasnacheev --- modules/yarn/pom.xml | 25 +++ .../org/apache/ignite/yarn/IgniteProvider.java | 217 +++-- .../org/apache/ignite/yarn/IgniteYarnClient.java | 12 +- modules/yarn/src/main/resources/ignite.properties | 23 +++ 4 files changed, 130 insertions(+), 147 deletions(-) diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml index 6c95a82..3d3642e 100644 --- a/modules/yarn/pom.xml +++ b/modules/yarn/pom.xml @@ -100,6 +100,31 @@ + +maven-resources-plugin +3.1.0 + + + copy-resources + validate + + copy-resources + + + src/main/resources + + + ../core/src/main/resources + + ignite.properties + + + + + + + + maven-assembly-plugin 2.4.1 diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java index d4faa38..311058c 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java @@ -21,33 +21,29 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.nio.channels.Channels; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; +import java.util.Properties; +import java.util.logging.Logger; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.ignite.yarn.utils.IgniteYarnUtils; /** - * Downloads and stores Ignite. + * Downloads and stores Ignite release. */ public class IgniteProvider { /** */ -public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php";; +public static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName()); /** */ -private ClusterProperties props; - -/** */ -private String latestVersion = null; +public static final String DOWNLOAD_LINK = "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/";; /** */ -private boolean hdfs = false; +private ClusterProperties props; /** */ private FileSystem fs; @@ -62,93 +58,67 @@ public class IgniteProvider { } /** - * @return Latest ignite version. + * @return Path to current Ignite release. */ public Path getIgnite() throws Exception { File folder = checkDownloadFolder(); -if (latestVersion == null) { -List localFiles = findIgnites(folder); -List hdfsFiles = findIgnites(fs, props.igniteReleasesDir()); +Properties verProps = new Properties(); -String localLatestVersion = findLatestVersion(localFiles); -String hdfsLatestVersion = findLatestVersion(hdfsFiles); +try (InputStream is = IgniteProvider.class.getClassLoader().getResourceAsStream("ignite.properties")) { +verProps.load(is); +} -if (localLatestVersion != null && hdfsLatestVersion != null) { -if (VersionComparator.INSTANCE.compare(hdfsLatestVersion, localLatestVersion) >= 0) { -latestVersion = hdfsLatestVersion; +String curVer = verProps.getProperty("ignite.version"); -hdfs = true; -} -} -else if (localLatestVersion != null) -latestVersion = localLatestVersion; -else if (hdfsLatestVersion != null) { -latestVersion = hdfsLat
[ignite] branch master updated: IGNITE-11911 Fixed GridToStringBuilder memory overconsumption - Fixes #6603.
This is an automated email from the ASF dual-hosted git repository. alexpl pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 8a6c0d4 IGNITE-11911 Fixed GridToStringBuilder memory overconsumption - Fixes #6603. 8a6c0d4 is described below commit 8a6c0d4a59f60e595801c2d20e764bb620335058 Author: Aleksey Plekhanov AuthorDate: Thu Jun 13 16:21:05 2019 +0300 IGNITE-11911 Fixed GridToStringBuilder memory overconsumption - Fixes #6603. --- .../internal/util/tostring/SBLimitedLength.java| 2 +- .../util/tostring/GridToStringBuilderSelfTest.java | 33 ++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java index 640555a..e18e0ed 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java @@ -290,7 +290,7 @@ public class SBLimitedLength extends GridStringBuilder { return super.toString(); else { int tailLen = tail.length(); -StringBuilder res = new StringBuilder(impl().capacity() + tailLen + 100); +StringBuilder res = new StringBuilder(impl().length() + tailLen + 100); res.append(impl()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java index 5e220fc..197d346 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.util.tostring; +import com.sun.management.ThreadMXBean; +import java.lang.management.ManagementFactory; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; @@ -554,6 +556,37 @@ public class GridToStringBuilderSelfTest extends GridCommonAbstractTest { } /** + * Test GridToStringBuilder memory consumption. + */ +@Test +public void testMemoryConsumption() { +int objCnt = 100; + +ThreadMXBean bean = (ThreadMXBean)ManagementFactory.getThreadMXBean(); + +TestClass2 obj = new TestClass2(new String(new char[1_000_000])); + +List arr = new ArrayList<>(objCnt); + +for (int i = 1; i <= objCnt; i++) +arr.add(new TestClass2(new String(new char[i]))); + +GridToStringBuilder.toString(TestClass2.class, obj); + +long allocated0 = bean.getThreadAllocatedBytes(Thread.currentThread().getId()); + +for (TestClass2 item : arr) +GridToStringBuilder.toString(TestClass2.class, item); + +long allocated1 = bean.getThreadAllocatedBytes(Thread.currentThread().getId()); + +log.info("Memory allocated by GridToStringBuilder for " + objCnt + " objects: " + (allocated1 - allocated0)); + +assertTrue("Too much memory allocated by GridToStringBuilder: " + (allocated1 - allocated0), +allocated1 - allocated0 < 1_000_000); +} + +/** * @param exp Expected. * @param w Wrapper. */
[ignite-teamcity-bot] branch master updated: TC Bot modules refactoring: JIRA module extracted, PR processor moved to the engine
This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git The following commit(s) were added to refs/heads/master by this push: new ef7ba95 TC Bot modules refactoring: JIRA module extracted, PR processor moved to the engine ef7ba95 is described below commit ef7ba95983cef95891f09bd334ff3556d69a7036 Author: Dmitriy Pavlov AuthorDate: Thu Jun 13 15:01:24 2019 +0300 TC Bot modules refactoring: JIRA module extracted, PR processor moved to the engine --- .../org/apache/ignite/ci/di/IgniteTcBotModule.java | 4 +- .../apache/ignite/ci/runners/ClientTmpHelper.java | 2 +- .../ignite/ci/tcbot/conf/JiraServerConfig.java | 2 +- .../tcbot/visa/TcBotTriggerAndSignOffService.java | 13 +- .../ignite/ci/web/rest/parms/FullQueryParams.java | 196 - .../ignite/ci/web/rest/pr/GetPrTestFailures.java | 2 +- .../rest/tracked/GetTrackedBranchTestResults.java | 4 +- .../ci/tcbot/chain/MockBasedTcBotModule.java | 8 +- .../ci/tcbot/chain/PrChainsProcessorTest.java | 2 + .../IgnitedTcInMemoryIntegrationTest.java | 2 +- settings.gradle| 2 + .../common/conf/IDataSourcesConfigSupplier.java| 2 + tcbot-engine/build.gradle | 2 +- .../tcbot/engine/pr}/BranchTicketMatcher.java | 31 ++-- .../ignite/tcbot/engine/pr}/PrChainsProcessor.java | 30 ++-- {tcbot-engine => tcbot-jira-ignited}/build.gradle | 3 +- .../ignite/ci/jira/ignited/TicketCompacted.java| 14 +- .../apache/ignite/jiraignited}/IJiraIgnited.java | 17 +- .../ignite/jiraignited}/IJiraIgnitedProvider.java | 2 +- .../apache/ignite/jiraignited}/JiraIgnited.java| 9 +- .../ignite/jiraignited}/JiraIgnitedModule.java | 4 +- .../ignite/jiraignited}/JiraIgnitedProvider.java | 6 +- .../apache/ignite/jiraignited}/JiraTicketDao.java | 6 +- .../apache/ignite/jiraignited}/JiraTicketSync.java | 20 +-- tcbot-jira/README.md | 3 + {tcbot-engine => tcbot-jira}/build.gradle | 7 +- .../org/apache/ignite/jiraservice}/Fields.java | 2 +- .../ignite/jiraservice}/IJiraIntegration.java | 3 +- .../jiraservice}/IJiraIntegrationProvider.java | 2 +- .../java/org/apache/ignite/jiraservice}/Jira.java | 7 +- .../ignite/jiraservice}/JiraIntegrationModule.java | 2 +- .../jiraservice}/JiraIntegrationProvider.java | 2 +- .../org/apache/ignite/jiraservice}/Status.java | 2 +- .../org/apache/ignite/jiraservice}/Ticket.java | 6 +- .../org/apache/ignite/jiraservice}/Tickets.java| 4 +- .../apache/ignite/jiraservice}/package-info.java | 2 +- .../tcbot/persistence}/scheduler/NamedTask.java| 2 +- .../persistence}/scheduler/SchedulerModule.java| 3 +- .../persistence}/scheduler/TcBotScheduler.java | 3 +- 39 files changed, 132 insertions(+), 301 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java index efed362..dcceff1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java @@ -28,9 +28,9 @@ import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.ci.db.Ignite1Init; import org.apache.ignite.ci.di.cache.GuavaCachedModule; -import org.apache.ignite.ci.di.scheduler.SchedulerModule; +import org.apache.ignite.tcbot.persistence.scheduler.SchedulerModule; import org.apache.ignite.githubignited.GitHubIgnitedModule; -import org.apache.ignite.ci.jira.ignited.JiraIgnitedModule; +import org.apache.ignite.jiraignited.JiraIgnitedModule; import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.observer.ObserverTask; import org.apache.ignite.ci.tcbot.TcBotBusinessServicesModule; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java index b114350..58a20b1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java @@ -23,7 +23,7 @@ import org.apache.ignite.ci.db.TcHelperDb; import org.apache.ignite.githubignited.IGitHubConnIgnited; import org.apache.ignite.ci.issue.Issue; import org.apache.ignite.ci.issue.IssuesStorage; -import org.apache.ignite.ci.jira.ignited.JiraTicketDao; +import org.apache.ignite.jiraignited.JiraTicketDao; /** * Utility class for local connection to TC helper DB (server) and any manipulations with data needed. diff --git a/ignite-tc-helper-web/src/main/java/org/apache
[ignite-teamcity-bot] branch master updated: TC Bot modules refactoring: GitHub module extracted
This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git The following commit(s) were added to refs/heads/master by this push: new 6a5572f TC Bot modules refactoring: GitHub module extracted 6a5572f is described below commit 6a5572fa8bfb910f6e670103664882bb7c85e61c Author: Dmitriy Pavlov AuthorDate: Thu Jun 13 11:53:19 2019 +0300 TC Bot modules refactoring: GitHub module extracted --- build.gradle | 12 +++-- ignite-tc-helper-web/build.gradle | 6 +-- .../java/org/apache/ignite/ci/db/TcHelperDb.java | 9 .../org/apache/ignite/ci/di/IgniteTcBotModule.java | 2 +- .../ignite/ci/jira/ignited/JiraTicketDao.java | 3 +- .../apache/ignite/ci/runners/ClientTmpHelper.java | 2 +- .../ignite/ci/tcbot/chain/PrChainsProcessor.java | 6 +-- .../ignite/ci/tcbot/visa/BranchTicketMatcher.java | 4 +- .../tcbot/visa/TcBotTriggerAndSignOffService.java | 4 +- .../apache/ignite/ci/web/rest/TriggerBuilds.java | 5 +- .../ignite/ci/web/rest/pr/GetPrTestFailures.java | 6 +-- .../ignite/ci/github/pure/GitHubPrsParseTest.java | 54 -- .../ci/tcbot/chain/MockBasedTcBotModule.java | 8 ++-- jetty-launcher/build.gradle| 6 +-- settings.gradle| 2 + tcbot-common/build.gradle | 15 ++ .../common/conf/IDataSourcesConfigSupplier.java| 2 + tcbot-engine/build.gradle | 8 +--- tcbot-github-ignited/README.md | 3 ++ .../build.gradle | 7 +-- .../githubignited}/GitHubConnIgnitedImpl.java | 14 +++--- .../ignite/githubignited}/GitHubIgnitedModule.java | 4 +- .../githubignited}/GitHubIgnitedProvImpl.java | 6 +-- .../ignite/githubignited}/IGitHubConnIgnited.java | 2 +- .../githubignited}/IGitHubConnIgnitedProvider.java | 2 +- tcbot-github/README.md | 3 ++ {tcbot-engine => tcbot-github}/build.gradle| 11 ++--- .../org/apache/ignite/ci/github/GitHubBranch.java | 3 +- .../apache/ignite/ci/github/GitHubBranchKey.java | 7 +-- .../apache/ignite/ci/github/GitHubBranchShort.java | 3 +- .../org/apache/ignite/ci/github/GitHubCommit.java | 3 +- .../org/apache/ignite/ci/github/GitHubUser.java| 3 +- .../org/apache/ignite/ci/github/PullRequest.java | 5 +- .../githubservice}/GitHubCachingProvider.java | 2 +- .../githubservice}/GitHubConnectionImpl.java | 11 ++--- .../githubservice}/GitHubIntegrationModule.java| 2 +- .../ignite/githubservice}/IGitHubConnection.java | 4 +- .../githubservice}/IGitHubConnectionProvider.java | 2 +- .../ignite/githubservice}/GitHubPrsParseTest.java | 3 +- tcbot-teamcity-ignited/build.gradle| 7 +-- tcbot-teamcity/build.gradle| 9 +--- 41 files changed, 98 insertions(+), 172 deletions(-) diff --git a/build.gradle b/build.gradle index 1c2ddc3..edf7b84 100644 --- a/build.gradle +++ b/build.gradle @@ -58,13 +58,19 @@ allprojects { slf4jVer = '1.7.25' gsonVer = '2.8.2' + +junitVer = '4.12' +mockitoVer = '2.22.0' } repositories { //uncomment for RC testing - // maven { - // url "https://repository.apache.org/content/repositories/orgapacheignite-1463/"; - // } +// maven { +// url "https://repository.apache.org/content/repositories/orgapacheignite-1463/"; +// } + +mavenCentral() +mavenLocal() } } diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle index 9f5c512..313f275 100644 --- a/ignite-tc-helper-web/build.gradle +++ b/ignite-tc-helper-web/build.gradle @@ -20,11 +20,7 @@ apply plugin: 'war' sourceCompatibility = '1.8' targetCompatibility = '1.8' - -repositories { -mavenCentral() -mavenLocal() -} + // https://www.apache.org/legal/resolved.html#category-a dependencies { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java index 14642cc..e706b33 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java @@ -111,15 +111,6 @@ public class TcHelperDb { } -@NotNull -public static CacheConfiguration getCache8PartsConfig(String name) { -CacheConfiguration ccfg = new CacheConfiguration<>(name); - -ccfg.setAffinity(new RendezvousAffinityFunction(false, 8)); - -return ccfg; -} - public static class LocalOnlyTcpDiscoveryIpFinder implements TcpDiscoveryIpFinder { /** Port. */ private int port; di