Repository: ignite Updated Branches: refs/heads/master a228c246a -> 02b59e433
ignite-1559: UriDeploymentHttpScanner tests and javadoc Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/02b59e43 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/02b59e43 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/02b59e43 Branch: refs/heads/master Commit: 02b59e433bce7a4c3eece7a80e7a053ae0d69373 Parents: a228c24 Author: Artem SHutak <ashu...@gridgain.com> Authored: Thu Oct 15 17:18:15 2015 -0700 Committer: Valentin Kulichenko <valentin.kuliche...@gmail.com> Committed: Thu Oct 15 17:18:15 2015 -0700 ---------------------------------------------------------------------- .../ignite/spi/deployment/DeploymentSpi.java | 8 +- modules/core/src/test/config/tests.properties | 3 + modules/extdata/uri/pom.xml | 11 +- .../spi/deployment/uri/UriDeploymentSpi.java | 93 ++++++++----- .../scanners/http/UriDeploymentHttpScanner.java | 10 +- .../http/GridHttpDeploymentSelfTest.java | 132 +++++++++++++++++-- 6 files changed, 204 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/core/src/main/java/org/apache/ignite/spi/deployment/DeploymentSpi.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/deployment/DeploymentSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/deployment/DeploymentSpi.java index 7a1f709..af09e48 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/deployment/DeploymentSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/deployment/DeploymentSpi.java @@ -46,13 +46,13 @@ import org.jetbrains.annotations.Nullable; * local deployment. * </strong> * <p> - * Ignite provides the following {@code GridDeploymentSpi} implementations: + * Ignite provides following {@code GridDeploymentSpi} implementations out of the box: * <ul> * <li>{@link org.apache.ignite.spi.deployment.local.LocalDeploymentSpi}</li> * <li>{@ignitelink org.apache.ignite.spi.deployment.uri.UriDeploymentSpi}</li> * </ul> - * <b>NOTE:</b> this SPI (i.e. methods in this interface) should never be used directly. SPIs provide - * internal view on the subsystem and is used internally by Ignite kernal. In rare use cases when + * <b>NOTE:</b> SPI methods should never be used directly. SPIs provide + * internal view on the subsystem and is used internally by Ignite. In rare use cases when * access to a specific implementation of this SPI is required - an instance of this SPI can be obtained * via {@link org.apache.ignite.Ignite#configuration()} method to check its configuration properties or call other non-SPI * methods. Note again that calling methods from this interface on the obtained instance can lead @@ -104,4 +104,4 @@ public interface DeploymentSpi extends IgniteSpi { * @param lsnr Listener for deployment events. {@code null} to unset the listener. */ public void setListener(@Nullable DeploymentListener lsnr); -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/core/src/test/config/tests.properties ---------------------------------------------------------------------- diff --git a/modules/core/src/test/config/tests.properties b/modules/core/src/test/config/tests.properties index ce6d4d8..49e616e 100644 --- a/modules/core/src/test/config/tests.properties +++ b/modules/core/src/test/config/tests.properties @@ -67,7 +67,10 @@ ant.gar.srcdir=@{IGNITE_HOME}/modules/extdata/uri/target/classes/ # GAR paths to use in URI deployment SPI tests ant.urideployment.gar.uri=file://freq=5000@localhost/EXTDATA/uri/target/deploy ant.urideployment.gar.file=modules/extdata/uri/target/deploy/uri.gar +ant.urideployment.gar.libs-file=modules/extdata/uri/target/deploy/uri-libs.gar +ant.urideployment.gar.classes-file=modules/extdata/uri/target/deploy/uri-classes.gar ant.urideployment.gar.path=modules/extdata/uri/target/deploy/ +ant.urideployment.gar.path.tmp=modules/extdata/uri/target/deploy_tmp/ # Classpath directory for GridP2PUserVersionChangeSelfTest ant.userversion.class.dir=@{IGNITE_HOME}/modules/tests/java/ http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/extdata/uri/pom.xml ---------------------------------------------------------------------- diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml index d9a9297..d5e6349 100644 --- a/modules/extdata/uri/pom.xml +++ b/modules/extdata/uri/pom.xml @@ -144,14 +144,23 @@ </fileset> </copy> - <copy file="${settings.localRepository}/com/sun/mail/javax.mail/1.5.2/javax.mail-1.5.2.jar" todir="${basedir}/target/classes/lib" /> + <copy file="${settings.localRepository}/com/sun/mail/javax.mail/1.5.2/javax.mail-1.5.2.jar" todir="${basedir}/target/libs" /> <zip destfile="${basedir}/target/classes/lib/depend.jar" encoding="UTF-8"> <zipfileset dir="modules/uri-dependency/target/classes" /> </zip> + <copy file="${basedir}/target/classes/lib/depend.jar" todir="${basedir}/target/libs" /> + + <mkdir dir="${basedir}/target/deploy_tmp/"/> + <taskdef name="gar" classname="org.apache.ignite.util.antgar.IgniteDeploymentGarAntTask" /> + <gar destfile="${basedir}/target/deploy/uri-classes.gar" basedir="${basedir}/target/classes" /> + <gar destfile="${basedir}/target/deploy/uri-libs.gar" basedir="${basedir}/target/libs" /> + + <copy file="${settings.localRepository}/com/sun/mail/javax.mail/1.5.2/javax.mail-1.5.2.jar" todir="${basedir}/target/classes/lib" /> + <gar destfile="${basedir}/target/deploy/uri.gar" basedir="${basedir}/target/classes" /> <!-- http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java index c48398d..5f65731 100644 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java @@ -80,7 +80,7 @@ import org.jetbrains.annotations.Nullable; * <p> * SPI tracks all changes of every given URI. This means that if any file is * changed or deleted, SPI will re-deploy or delete corresponding tasks. - * Note that the very first apply to {@link #findResource(String)} findClassLoader(String)} + * Note that the very first apply to {@link #findResource(String)} * is blocked until SPI finishes scanning all URI's at least once. * <p> * There are several deployable unit types supported: @@ -145,11 +145,33 @@ import org.jetbrains.annotations.Nullable; * URI 'path' field will be automatically encoded. By default this flag is * set to {@code true}. * <p> + * <h1 class="header">Code Example</h1> + * The following example demonstrates how the deployment SPI can be used. It expects that you have a GAR file + * in 'home/username/ignite/work/my_deployment/file' folder which contains 'myproject.HelloWorldTask' class. + * <pre name="code" class="java"> + * IgniteConfiguration cfg = new IgniteConfiguration(); + * + * DeploymentSpi deploymentSpi = new UriDeploymentSpi(); + * + * deploymentSpi.setUriList(Arrays.asList("file:///home/username/ignite/work/my_deployment/file")); + * + * cfg.setDeploymentSpi(deploymentSpi); + * + * try (Ignite ignite = Ignition.start(cfg)) { + * ignite.compute().execute("myproject.HelloWorldTask", "my args"); + * } + * </pre> * <h1 class="header">Configuration</h1> * {@code UriDeploymentSpi} has the following optional configuration * parameters (there are no mandatory parameters): * <ul> * <li> + * Array of {@link UriDeploymentScanner}-s which will be used to deploy resources + * (see {@link #setScanners(UriDeploymentScanner...)}). If not specified, preconfigured {@link UriDeploymentFileScanner} + * and {@link UriDeploymentHttpScanner} are used. You can implement your own scanner + * by implementing {@link UriDeploymentScanner} interface. + * </li> + * <li> * Temporary directory path where scanned GAR files and directories are * copied to (see {@link #setTemporaryDirectoryPath(String) setTemporaryDirectoryPath(String)}). * </li> @@ -163,25 +185,28 @@ import org.jetbrains.annotations.Nullable; * </li> * </ul> * <h1 class="header">Protocols</h1> - * Following protocols are supported in SPI: + * Following protocols are supported by this SPI out of the box: * <ul> * <li><a href="#file">file://</a> - File protocol</li> - * <li><a href="#classes">classes://</a> - Custom File protocol.</li> * <li><a href="#http">http://</a> - HTTP protocol</li> * <li><a href="#http">https://</a> - Secure HTTP protocol</li> * </ul> + * <strong>Custom Protocols.</strong> + * <p> + * You can add support for additional protocols if needed. To do this implement UriDeploymentScanner interface and + * plug your implementation into the SPI via {@link #setScanners(UriDeploymentScanner...)} method. + * <p> * In addition to SPI configuration parameters, all necessary configuration * parameters for selected URI should be defined in URI. Different protocols * have different configuration parameters described below. Parameters are * separated by '{@code ;}' character. * <p> - * <a name="file"></a> * <h1 class="header">File</h1> * For this protocol SPI will scan folder specified by URI on file system and * download any GAR files or directories that end with .gar from source * directory defined in URI. For file system URI must have scheme equal to {@code file}. * <p> - * Following parameters are supported for FILE protocol: + * Following parameters are supported: * <table class="doctable"> * <tr> * <th>Parameter</th> @@ -189,24 +214,29 @@ import org.jetbrains.annotations.Nullable; * <th>Optional</th> * <th>Default</th> * </tr> + * <tr> + * <td>freq</td> + * <td>Scanning frequency in milliseconds.</td> + * <td>Yes</td> + * <td>{@code 5000} ms specified in {@link UriDeploymentFileScanner#DFLT_SCAN_FREQ}.</td> + * </tr> * </table> * <h2 class="header">File URI Example</h2> * The following example will scan {@code 'c:/Program files/ignite/deployment'} - * folder on local box every {@code '5000'} milliseconds. Note that since path + * folder on local box every {@code '1000'} milliseconds. Note that since path * has spaces, {@link #setEncodeUri(boolean) setEncodeUri(boolean)} parameter must * be set to {@code true} (which is default behavior). * <blockquote class="snippet"> - * {@code file://freq=5000@localhost/c:/Program files/ignite/deployment} + * {@code file://freq=2000@localhost/c:/Program files/ignite/deployment} * </blockquote> * <a name="classes"></a> - * <h1 class="header">Classes</h1> - * For this protocol SPI will scan folder specified by URI on file system - * looking for compiled classes that implement {@link org.apache.ignite.compute.ComputeTask} interface. - * This protocol comes very handy during development, as it allows developer - * to specify IDE compilation output folder as URI and all task classes - * in that folder will be deployed automatically. + * <h2 class="header">HTTP/HTTPS</h2> + * URI deployment scanner tries to read DOM of the html it points to and parses out href attributes of all <a> tags + * - this becomes the collection of URLs to GAR files that should be deployed. It's important that HTTP scanner + * uses {@code URLConnection.getLastModified()} method to check if there were any changes since last iteration + * for each GAR-file before redeploying. * <p> - * Following parameters are supported for CLASSES protocol: + * Following parameters are supported: * <table class="doctable"> * <tr> * <th>Parameter</th> @@ -214,20 +244,17 @@ import org.jetbrains.annotations.Nullable; * <th>Optional</th> * <th>Default</th> * </tr> + * <tr> + * <td>freq</td> + * <td>Scanning frequency in milliseconds.</td> + * <td>Yes</td> + * <td>{@code 300000} ms specified in {@link UriDeploymentHttpScanner#DFLT_SCAN_FREQ}.</td> + * </tr> * </table> - * <h2 class="header">Classes URI Example</h2> - * The following example will scan {@code 'c:/Program files/ignite/deployment'} - * folder on local box every {@code '5000'} milliseconds. Note that since path - * has spaces, {@link #setEncodeUri(boolean) setEncodeUri(boolean)} parameter must - * be set to {@code true} (which is default behavior). - * <blockquote class="snippet"> - * {@code classes://freq=5000@localhost/c:/Program files/ignite/deployment} - * </blockquote> - * <a name="http"></a> * <h2 class="header">HTTP URI Example</h2> - * The following example will scan {@code 'ignite/deployment'} folder with - * on site {@code 'www.mysite.com'} using authentication - * {@code 'username:password'} every {@code '10000'} milliseconds. + * The following example will download the page `www.mysite.com/ignite/deployment`, parse it and download and deploy + * all GAR files specified by href attributes of <a> elements on the page using authentication + * {@code 'username:password'} every '10000' milliseconds (only new/updated GAR-s). * <blockquote class="snippet"> * {@code http://username:password;freq=10...@www.mysite.com:110/ignite/deployment} * </blockquote> @@ -238,14 +265,9 @@ import org.jetbrains.annotations.Nullable; * * IgniteConfiguration cfg = new IgniteConfiguration(); * - * List<String> uris = new ArrayList<String>(5); - * - * uris.add("http://www.site.com/tasks";); - * uris.add("file://freq=20000@localhost/c:/Program files/gg-deployment"); - * uris.add("classes:///c:/Java_Projects/myproject/out"); - * * // Set URIs. - * deploySpi.setUriList(uris); + * deploySpi.setUriList(Arrays.asList("http://www.site.com/tasks";, + * "file://freq=20000@localhost/c:/Program files/gg-deployment")); * * // Override temporary directory path. * deploySpi.setTemporaryDirectoryPath("c:/tmp/grid"); @@ -254,7 +276,7 @@ import org.jetbrains.annotations.Nullable; * cfg.setDeploymentSpi(deploySpi); * * // Start grid. - * G.start(cfg); + * Ignition.start(cfg); * </pre> * <p> * <h2 class="header">Spring Example</h2> @@ -269,7 +291,6 @@ import org.jetbrains.annotations.Nullable; * <list> * <value>http://www.site.com/tasks</value> * <value>file://freq=20000@localhost/c:/Program files/gg-deployment</value> - * <value>classes:///c:/Java_Projects/myproject/out</value> * </list> * </property> * </bean> @@ -1325,4 +1346,4 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, @Override public String toString() { return S.toString(UriDeploymentSpi.class, this); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java index ef29752..48bfd7f 100644 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java @@ -60,11 +60,15 @@ import org.w3c.dom.NodeList; import org.w3c.tidy.Tidy; /** - * URI deployment HTTP scanner. + * HTTP-based URI deployment scanner. + * <p> + * This scanner reads DOM of the HTML available via {@link UriDeploymentScannerContext#getUri()} + * and parses out href attributes of all {@code <a>} tags - + * they become the collection of URLs to GAR files that should be deployed. */ public class UriDeploymentHttpScanner implements UriDeploymentScanner { /** Default scan frequency. */ - private static final int DFLT_SCAN_FREQ = 300000; + public static final int DFLT_SCAN_FREQ = 300000; /** Secure socket protocol to use. */ private static final String PROTOCOL = "TLS"; @@ -501,4 +505,4 @@ public class UriDeploymentHttpScanner implements UriDeploymentScanner { return true; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/02b59e43/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridHttpDeploymentSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridHttpDeploymentSelfTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridHttpDeploymentSelfTest.java index 216d0ab..c0044c3 100644 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridHttpDeploymentSelfTest.java +++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridHttpDeploymentSelfTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.spi.deployment.uri.scanners.http; +import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.List; import javax.servlet.http.HttpServletResponse; @@ -38,8 +40,35 @@ import static org.eclipse.jetty.http.HttpHeader.LAST_MODIFIED; */ @GridSpiTest(spi = UriDeploymentSpi.class, group = "Deployment SPI") public class GridHttpDeploymentSelfTest extends GridUriDeploymentAbstractSelfTest { + /** Frequency */ + private static final int FREQ = 5000; + + /** */ + public static final String LIBS_GAR = "libs-file.gar"; + + /** */ + public static final String CLASSES_GAR = "classes-file.gar"; + + /** */ + public static final String ALL_GAR = "file.gar"; + + /** Gar-file which contains libs. */ + public static final String LIBS_GAR_FILE_PATH = U.resolveIgnitePath( + GridTestProperties.getProperty("ant.urideployment.gar.libs-file")).getPath(); + + /** Gar-file which contains classes (cannot be used without libs). */ + public static final String CLASSES_GAR_FILE_PATH = U.resolveIgnitePath( + GridTestProperties.getProperty("ant.urideployment.gar.classes-file")).getPath(); + + /** Gar-file which caontains both libs and classes. */ + public static final String ALL_GAR_FILE_PATH = U.resolveIgnitePath( + GridTestProperties.getProperty("ant.urideployment.gar.file")).getPath(); + /** Jetty. */ - private Server srv; + private static Server srv; + + /** Resource base. */ + private static String rsrcBase; /** {@inheritDoc} */ @Override protected void beforeSpiStarted() throws Exception { @@ -60,8 +89,12 @@ public class GridHttpDeploymentSelfTest extends GridUriDeploymentAbstractSelfTes }; hnd.setDirectoriesListed(true); - hnd.setResourceBase( - U.resolveIgnitePath(GridTestProperties.getProperty("ant.urideployment.gar.path")).getPath()); + + File resourseBaseDir = U.resolveIgnitePath(GridTestProperties.getProperty("ant.urideployment.gar.path.tmp")); + + rsrcBase = resourseBaseDir.getPath(); + + hnd.setResourceBase(rsrcBase); srv.setHandler(hnd); @@ -82,11 +115,63 @@ public class GridHttpDeploymentSelfTest extends GridUriDeploymentAbstractSelfTes } /** - * @throws Exception if failed. + * @throws Exception If failed. */ - public void testDeployment() throws Exception { - checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); - checkTask("GridUriDeploymentTestWithNameTask3"); + public void testDeployUndeploy2Files() throws Exception { + checkNoTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + + try { + copyToResourceBase(LIBS_GAR_FILE_PATH, LIBS_GAR); + + copyToResourceBase(CLASSES_GAR_FILE_PATH, CLASSES_GAR); + + Thread.sleep(FREQ + 3000); + + checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + deleteFromResourceBase(LIBS_GAR); + deleteFromResourceBase(CLASSES_GAR); + + Thread.sleep(FREQ + 3000); + + checkNoTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + } + } + + /** + * @throws Exception If failed. + */ + public void testSameContantFiles() throws Exception { + checkNoTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + + try { + copyToResourceBase(ALL_GAR_FILE_PATH, ALL_GAR); + + Thread.sleep(FREQ + 3000); + + checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + + copyToResourceBase(ALL_GAR_FILE_PATH, "file-copy.gar"); + + Thread.sleep(FREQ + 3000); + + checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + } + catch (Throwable e) { + e.printStackTrace(); + } + finally { + deleteFromResourceBase(ALL_GAR); + deleteFromResourceBase("file-copy.gar"); + + Thread.sleep(FREQ + 3000); + + checkNoTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); + } } /** @@ -94,6 +179,35 @@ public class GridHttpDeploymentSelfTest extends GridUriDeploymentAbstractSelfTes */ @GridSpiTestConfig public List<String> getUriList() { - return Collections.singletonList("http://freq=5000@localhost:8080/";); + return Collections.singletonList("http://freq="+FREQ+"@localhost:8080/";); + } + + /** + * @param fileName File name. + */ + private void deleteFromResourceBase(String fileName) { + File file = new File(rsrcBase + '/' + fileName); + + if (!file.delete()) + U.warn(log, "Could not delete file: " + file); + } + + /** + * @param path Path to the file which should be copied. + * @param newFileName New file name. + * @throws IOException If exception. + */ + private void copyToResourceBase(String path, String newFileName) throws IOException { + File file = new File(path); + + assert file.exists() : "Test file not found [path=" + path + ']'; + + File newFile = new File(rsrcBase + '/' + newFileName); + + assert !newFile.exists(); + + U.copy(file, newFile, false); + + assert newFile.exists(); } -} \ No newline at end of file +}
