This is an automated email from the ASF dual-hosted git repository. sdedic pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push: new ef50acf NETBEANS-6307: check global artifact cache, ignore per-project cached structure if not consistent. (#3375) ef50acf is described below commit ef50acf5dfa975543c966e07ac6145ced7478090 Author: Svatopluk Dedic <svatopluk.de...@oracle.com> AuthorDate: Mon Dec 20 16:58:29 2021 +0100 NETBEANS-6307: check global artifact cache, ignore per-project cached structure if not consistent. (#3375) NETBEANS-6307: check global artifact cache, ignore per-project cached structure if not consistent. --- .../gradle/loaders/DiskCacheProjectLoader.java | 2 +- .../gradle/loaders/GradleArtifactStore.java | 63 +++++++- .../gradle/loaders/DiskCacheProjectLoaderTest.java | 169 +++++++++++++++++++++ .../gradle/loaders/testReloadProject.gradle | 29 ++++ 4 files changed, 260 insertions(+), 3 deletions(-) diff --git a/extide/gradle/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoader.java b/extide/gradle/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoader.java index 66ee2da..db2238b 100644 --- a/extide/gradle/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoader.java +++ b/extide/gradle/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoader.java @@ -42,7 +42,7 @@ public class DiskCacheProjectLoader extends AbstractProjectLoader { if (cache.isCompatible()) { GradleProject prev = createGradleProject(cache.loadData()); LOG.log(Level.FINER, "Loaded from cache: {0}, valid: {1}", new Object[] { prev, cache.isValid() }); - if (cache.isValid()) { + if (cache.isValid() && GradleArtifactStore.getDefault().sanityCheckCachedProject(prev)) { updateSubDirectoryCache(prev); return prev; } diff --git a/extide/gradle/src/org/netbeans/modules/gradle/loaders/GradleArtifactStore.java b/extide/gradle/src/org/netbeans/modules/gradle/loaders/GradleArtifactStore.java index 9dc2729..0b4119d 100644 --- a/extide/gradle/src/org/netbeans/modules/gradle/loaders/GradleArtifactStore.java +++ b/extide/gradle/src/org/netbeans/modules/gradle/loaders/GradleArtifactStore.java @@ -28,12 +28,18 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.event.ChangeListener; +import org.netbeans.modules.gradle.GradleModuleFileCache21; import org.netbeans.modules.gradle.GradleProject; import org.openide.modules.OnStart; import org.openide.modules.Places; @@ -45,7 +51,8 @@ import org.openide.util.RequestProcessor; * @author Laszlo Kishalmi */ public class GradleArtifactStore { - + private static Logger LOG = Logger.getLogger(GradleArtifactStore.class.getName()); + private static final String GRADLE_ARTIFACT_STORE_INFO = "gradle/artifact-store-info.ser"; public static final RequestProcessor RP = new RequestProcessor("Gradle Artifact Store", 1); //NOI18 @@ -114,11 +121,13 @@ public class GradleArtifactStore { if (gp.getQuality().worseThan(Quality.FULL)) { return; } + List<String> gavs = new ArrayList<>(); boolean changed = false; for (GradleConfiguration conf : gp.getBaseProject().getConfigurations().values()) { for (GradleDependency.ModuleDependency module : conf.getModules()) { Set<File> oldBins = binaries.get(module.getId()); Set<File> newBins = module.getArtifacts(); + gavs.add(module.getId()); if (oldBins != newBins) { binaries.put(module.getId(), newBins); changed = true; @@ -142,6 +151,9 @@ public class GradleArtifactStore { } } } + LOG.log(Level.FINE, "Cache refresh for project {0}, changed {2}, module deps {1}", new Object[] { + gp.getBaseProject().getProjectDir(), gavs, changed + }); if (changed) { store(); notifyTask.schedule(1000); @@ -159,7 +171,54 @@ public class GradleArtifactStore { store(); } } - + + /** + * Checks that all dependencies the project thinks should be in the global cache + * are actually in the global cache. If the global artifact cache does not contain + * an entry from a resolved dependency in the project cache then many random failures can + * occur, as an artifact is formally OK, but its JAR cannot be looked up. + * + * @param gp cached project + * @return true if the cached project resolves. + */ + public final boolean sanityCheckCachedProject(GradleProject gp) { + GradleModuleFileCache21 modCache = GradleModuleFileCache21.getGradleFileCache(); + for (GradleConfiguration conf : gp.getBaseProject().getConfigurations().values()) { + for (GradleDependency.ModuleDependency module : conf.getModules()) { + Set<File> oldBins = binaries.get(module.getId()); + if (oldBins == null || oldBins.isEmpty()) { + LOG.log(Level.FINE, "Checking {0}: Module dependency {1} not found in cache.", new Object[] { gp.getBaseProject().getProjectDir(), module.getId() }); + return false; + } + if (oldBins.size() == 1) { + File binary = oldBins.iterator().next(); + GradleModuleFileCache21.CachedArtifactVersion cav = modCache.resolveModule(module.getId()); + if (cav == null) { + LOG.log(Level.FINE, "Checking {0}: Cached artifact not found for {1}", new Object[] { gp.getBaseProject().getProjectDir(), module.getId() }); + return false; + } + GradleModuleFileCache21.CachedArtifactVersion.Entry javadocEntry = cav.getJavaDoc(); + GradleModuleFileCache21.CachedArtifactVersion.Entry sourceEntry = cav.getSources(); + if (sourceEntry != null && Files.exists(sourceEntry.getPath())) { + File check = sources.get(binary); + if (check == null || !check.toPath().equals(sourceEntry.getPath())) { + LOG.log(Level.FINE, "Checking {0}: cache does not list CachedArtifact for source {2}", new Object[] { gp.getBaseProject().getProjectDir(), module.getId(), sourceEntry.getPath() }); + return false; + } + } + if (javadocEntry != null && Files.exists(javadocEntry.getPath())) { + File check = javadocs.get(binary); + if (check == null || !check.toPath().equals(javadocEntry.getPath())) { + LOG.log(Level.FINE, "Checking {0}: cache does not list CachedArtifact for javadoc {2}", new Object[] { gp.getBaseProject().getProjectDir(), module.getId(), javadocEntry.getPath() }); + return false; + } + } + } + } + } + return true; + } + public final void addChangeListener(ChangeListener l) { cs.addChangeListener(l); } diff --git a/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoaderTest.java b/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoaderTest.java new file mode 100644 index 0000000..426d45b --- /dev/null +++ b/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/DiskCacheProjectLoaderTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.gradle.loaders; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import junit.framework.Test; +import junit.framework.TestSuite; +import org.netbeans.api.project.Project; +import org.netbeans.api.project.ProjectManager; +import org.netbeans.junit.NbModuleSuite; +import org.netbeans.modules.gradle.AbstractGradleProjectTestCase; +import org.netbeans.modules.gradle.api.GradleBaseProject; +import org.netbeans.modules.gradle.api.GradleConfiguration; +import org.netbeans.modules.gradle.api.GradleDependency.ModuleDependency; +import org.netbeans.modules.projectapi.SimpleFileOwnerQueryImplementation; +import org.netbeans.modules.projectapi.nb.NbProjectManager; +import org.netbeans.spi.project.ActionProgress; +import org.netbeans.spi.project.ActionProvider; +import org.netbeans.spi.project.ProjectFactory; +import org.netbeans.spi.project.ProjectManagerImplementation; +import org.openide.filesystems.FileObject; +import org.openide.filesystems.FileUtil; +import org.openide.util.Lookup; +import org.openide.util.lookup.Lookups; + +/** + * + * @author sdedic + */ +public class DiskCacheProjectLoaderTest extends AbstractGradleProjectTestCase{ + private FileObject projectDir; + + public DiskCacheProjectLoaderTest(String name) { + super(name); + } + + public static TestSuite suite() { + TestSuite ts = new TestSuite(); + NbModuleSuite.Configuration conf = NbModuleSuite.emptyConfiguration(). + addTest(DiskCacheProjectLoaderTest.class). + addTest("testInitialLoadProject"). + reuseUserDir(false). + gui(false). + enableModules(".*gradle.*"). + reuseUserDir(false); + + Test run1 = NbModuleSuite.create(conf); + + conf = NbModuleSuite.emptyConfiguration(). + addTest(DiskCacheProjectLoaderTest.class). + addTest("testLoadCachedProject"). + reuseUserDir(false). + gui(false). + enableModules(".*gradle.*"). + reuseUserDir(false); + + Test run2 = NbModuleSuite.create(conf); + ts.addTest(run1); + ts.addTest(run2); + + return ts; + } + + public void testLoadCachedProject() throws Exception { + GradleBaseProject gbp = openBaseProject(); + assertCompressHasArtifacts(gbp); + } + + private FileObject createProject(String name, String resource) throws IOException { + FileObject ret = FileUtil.toFileObject(getWorkDir()); + projectDir = ret.createFolder(name); + projectDir.getFileSystem().runAtomicAction(() -> {; + FileObject bs = projectDir.createData("build.gradle"); + try (OutputStream os = bs.getOutputStream()) { + FileUtil.copy(getClass().getResourceAsStream(resource), os); + } + }); + return projectDir; + } + + private GradleBaseProject openBaseProject() throws Exception { + String s = System.getProperty("gradle.test.project.path"); + FileObject a = FileUtil.toFileObject(new File(s)); + assertNotNull(a); + + // HACK HACK HACK: If the following is uncommented, a race condition happens + // between the test main thread and Git module that runs SimpleFileOwnerQueryImplementation on + // a project directory before the 'build.gradle' has been materialized, caching 'no project' answer until + // FS events are delivered & processed by project system (that reset the cache). + // + // Remove the hack after NETBEANS-6305 is fixed. + SimpleFileOwnerQueryImplementation.reset(); + Method m = NbProjectManager.class.getDeclaredMethod("reset"); + m.setAccessible(true); + m.invoke(Lookup.getDefault().lookup(ProjectManagerImplementation.class)); + // END HACK + + Object[] arr = Lookup.getDefault().lookupAll(ProjectFactory.class).toArray(); + Project proj = ProjectManager.getDefault().findProject(a); + assertNotNull(proj); + + ActionProvider ap = proj.getLookup().lookup(ActionProvider.class); + assertNotNull(ap); + + final CountDownLatch primeLatch = new CountDownLatch(1); + AtomicBoolean status = new AtomicBoolean(false); + ActionProgress prog = new ActionProgress() { + @Override + protected void started() { + } + + @Override + public void finished(boolean success) { + status.set(success); + primeLatch.countDown(); + } + }; + ap.invokeAction(ActionProvider.COMMAND_PRIME, Lookups.fixed(prog)); + primeLatch.await(); + + assertTrue(status.get()); + GradleBaseProject gbp = GradleBaseProject.get(proj); + + return gbp; + } + + private void assertCompressHasArtifacts(GradleBaseProject gbp) { + GradleConfiguration compC = gbp.getConfigurations().get("compileClasspath"); + Set<ModuleDependency> moDep = compC.findModules("org.apache.commons:commons-compress:1.20"); + assertNotNull(moDep != null); + assertEquals(1, moDep.size()); + + ModuleDependency compress = moDep.iterator().next(); + Set<File> artifacts = compress.getArtifacts(); + assertFalse(artifacts.isEmpty()); + } + + public void testInitialLoadProject() throws Exception { + FileObject a = createProject("projectA", "testReloadProject.gradle"); + + // The other test will run in a different NbModuleSuite, loaded by a different ClassLoader -- sharing in a static field is not possible. + System.setProperty("gradle.test.project.path", FileUtil.toFile(a).toString()); + + GradleBaseProject gbp = openBaseProject(); + assertCompressHasArtifacts(gbp); + } +} diff --git a/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/testReloadProject.gradle b/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/testReloadProject.gradle new file mode 100644 index 0000000..dfe75cc --- /dev/null +++ b/extide/gradle/test/unit/src/org/netbeans/modules/gradle/loaders/testReloadProject.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'java' + +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.apache.commons:commons-compress:1.20' + implementation 'javax.annotation:javax.annotation-api:1.3.2' +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists