Crate entity improvements * Allows configuration via file * Exposes more sensors. Uses SERVER_OK in SERVICE_NOT_UP_INDICATORS. * Adds integration tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/743fd2d0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/743fd2d0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/743fd2d0 Branch: refs/heads/master Commit: 743fd2d008d474157a0685357a8805d64b70be78 Parents: 35588b8 Author: Sam Corbett <[email protected]> Authored: Tue Jan 27 10:54:50 2015 +0000 Committer: Sam Corbett <[email protected]> Committed: Tue Jan 27 10:54:50 2015 +0000 ---------------------------------------------------------------------- software/database/pom.xml | 1 + .../entity/database/crate/CrateNode.java | 62 +++++++++++++-- .../entity/database/crate/CrateNodeDriver.java | 18 +++++ .../entity/database/crate/CrateNodeImpl.java | 82 ++++++++++++++------ .../database/crate/CrateNodeSshDriver.java | 53 ++++++++++--- .../brooklyn/entity/database/crate/crate.yaml | 28 +++++++ .../crate/CrateNodeIntegrationTest.java | 64 +++++++++++++++ 7 files changed, 269 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/pom.xml ---------------------------------------------------------------------- diff --git a/software/database/pom.xml b/software/database/pom.xml index 89090a1..593e923 100644 --- a/software/database/pom.xml +++ b/software/database/pom.xml @@ -46,6 +46,7 @@ the given components. These are files "without any degree of creativity" from the perspective of the Brooklyn/Apache contribution. --> + <exclude>src/main/resources/brooklyn/entity/database/crate/crate.yaml</exclude> <exclude>src/main/resources/brooklyn/entity/database/mariadb/my.cnf</exclude> <exclude>src/main/resources/brooklyn/entity/database/mysql/mysql.conf</exclude> <exclude>src/main/resources/brooklyn/entity/database/postgresql/postgresql.conf</exclude> http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/main/java/brooklyn/entity/database/crate/CrateNode.java ---------------------------------------------------------------------- diff --git a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNode.java b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNode.java index 4ee28c3..7f1dc3d 100644 --- a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNode.java +++ b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNode.java @@ -1,9 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package brooklyn.entity.database.crate; import brooklyn.config.ConfigKey; import brooklyn.entity.basic.Attributes; import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.SoftwareProcess; +import brooklyn.entity.database.DatastoreMixins.DatastoreCommon; import brooklyn.entity.java.UsesJava; import brooklyn.entity.java.UsesJavaMXBeans; import brooklyn.entity.java.UsesJmx; @@ -11,11 +30,13 @@ import brooklyn.entity.proxying.ImplementedBy; import brooklyn.event.AttributeSensor; import brooklyn.event.basic.AttributeSensorAndConfigKey; import brooklyn.event.basic.BasicAttributeSensorAndConfigKey; +import brooklyn.event.basic.PortAttributeSensorAndConfigKey; import brooklyn.event.basic.Sensors; +import brooklyn.location.basic.PortRanges; import brooklyn.util.flags.SetFromFlag; @ImplementedBy(CrateNodeImpl.class) -public interface CrateNode extends SoftwareProcess, UsesJava,UsesJmx, UsesJavaMXBeans { +public interface CrateNode extends SoftwareProcess, UsesJava,UsesJmx, UsesJavaMXBeans, DatastoreCommon { @SetFromFlag("version") ConfigKey<String> SUGGESTED_VERSION = ConfigKeys.newConfigKeyWithDefault(SoftwareProcess.SUGGESTED_VERSION, @@ -26,15 +47,44 @@ public interface CrateNode extends SoftwareProcess, UsesJava,UsesJmx, UsesJavaMX Attributes.DOWNLOAD_URL, "https://cdn.crate.io/downloads/releases/crate-${version}.tar.gz";); - AttributeSensor<String> MANAGEMENT_URI = Sensors.newStringSensor( + @SetFromFlag("serverConfig") + BasicAttributeSensorAndConfigKey<String> SERVER_CONFIG_URL = new BasicAttributeSensorAndConfigKey.StringAttributeSensorAndConfigKey( + "crate.serverConfig", "A URL of a YAML file to use to configure the server", + "classpath://brooklyn/entity/database/crate/crate.yaml"); + + @SetFromFlag("port") + public static final PortAttributeSensorAndConfigKey CRATE_PORT = new PortAttributeSensorAndConfigKey( + "crate.port", "The port for node-to-node communication", PortRanges.fromString("4300+")); + + @SetFromFlag("httpPort") + public static final PortAttributeSensorAndConfigKey CRATE_HTTP_PORT = new PortAttributeSensorAndConfigKey( + "crate.httpPort", "The port for HTTP traffic", PortRanges.fromString("4200+")); + + AttributeSensor<String> MANAGEMENT_URL = Sensors.newStringSensor( "crate.managementUri", "The address at which the Crate server listens"); AttributeSensor<String> SERVER_NAME = Sensors.newStringSensor( - "crate.serverName", "The name of the server"); + "crate.server.name", "The name of the server"); + + AttributeSensor<Boolean> SERVER_OK = Sensors.newBooleanSensor( + "crate.server.ok", "True if the server reports thus"); AttributeSensor<Integer> SERVER_STATUS = Sensors.newIntegerSensor( - "create.serverStatus", "The status of the server"); + "crate.server.status", "The status of the server"); AttributeSensor<String> SERVER_BUILD_TIMESTAMP = Sensors.newStringSensor( - "create.serverBuildTimestamp", "The Timestamp of the server build"); -} + "crate.server.buildTimestamp", "The timestamp of the server build"); + + AttributeSensor<String> SERVER_BUILD_HASH = Sensors.newStringSensor( + "crate.server.buildHash", "The build hash of the server"); + + AttributeSensor<Boolean> SERVER_IS_BUILD_SNAPSHOT = Sensors.newBooleanSensor( + "crate.server.isBuildSnapshot", "True if the server reports it is a snapshot build"); + + AttributeSensor<String> SERVER_LUCENE_VERSION = Sensors.newStringSensor( + "crate.server.luceneVersion", "The Lucene version of the server"); + + AttributeSensor<String> SERVER_ES_VERSION = Sensors.newStringSensor( + "crate.server.esVersion", "The ES version of the server"); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeDriver.java ---------------------------------------------------------------------- diff --git a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeDriver.java b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeDriver.java index af35858..225db07 100644 --- a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeDriver.java +++ b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeDriver.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package brooklyn.entity.database.crate; import brooklyn.entity.java.JavaSoftwareProcessDriver; http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeImpl.java ---------------------------------------------------------------------- diff --git a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeImpl.java b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeImpl.java index 72a11e6..5dcfc30 100644 --- a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeImpl.java +++ b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeImpl.java @@ -1,22 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package brooklyn.entity.database.crate; +import brooklyn.config.render.RendererHints; +import brooklyn.enricher.Enrichers; +import brooklyn.entity.basic.Attributes; import brooklyn.entity.basic.SoftwareProcessImpl; import brooklyn.entity.java.JavaAppUtils; import brooklyn.event.feed.http.HttpFeed; import brooklyn.event.feed.http.HttpPollConfig; import brooklyn.event.feed.http.HttpValueFunctions; import brooklyn.event.feed.jmx.JmxFeed; - +import brooklyn.util.guava.Functionals; public class CrateNodeImpl extends SoftwareProcessImpl implements CrateNode{ private JmxFeed jmxFeed; private HttpFeed httpFeed; - private static final int CRATE_PORT = 4200; - static { JavaAppUtils.init(); + RendererHints.register(MANAGEMENT_URL, RendererHints.namedActionWithUrl()); } @Override @@ -25,39 +45,55 @@ public class CrateNodeImpl extends SoftwareProcessImpl implements CrateNode{ } @Override - protected void disconnectSensors() { - disconnectServiceUpIsRunning(); - jmxFeed.stop(); - httpFeed.stop(); - super.disconnectSensors(); - } - - @Override protected void connectSensors() { super.connectSensors(); connectServiceUpIsRunning(); - jmxFeed = getJmxFeed(); - String uri = "http://"; + getAttribute(HOSTNAME) + ":" + CRATE_PORT; - setAttribute(MANAGEMENT_URI, uri); + jmxFeed = JavaAppUtils.connectMXBeanSensors(this); + setAttribute(DATASTORE_URL, "crate://" + getAttribute(HOSTNAME) + ":" + getPort()); + String url = "http://"; + getAttribute(HOSTNAME) + ":" + getHttpPort(); + setAttribute(MANAGEMENT_URL, url); httpFeed = HttpFeed.builder() .entity(this) - .baseUri(uri) - .poll(new HttpPollConfig<String>(CrateNode.SERVER_NAME) + .baseUri(url) + .poll(new HttpPollConfig<String>(SERVER_NAME) .onSuccess(HttpValueFunctions.jsonContents("name", String.class))) - .poll(new HttpPollConfig<Integer>(CrateNode.SERVER_STATUS) + .poll(new HttpPollConfig<Integer>(SERVER_STATUS) .onSuccess(HttpValueFunctions.jsonContents("status", Integer.class))) - .poll(new HttpPollConfig<String>(CrateNode.SERVER_BUILD_TIMESTAMP) - .onSuccess(HttpValueFunctions.jsonContents(new String[] {"version", "build_timestamp"}, String.class))) + .poll(new HttpPollConfig<Boolean>(SERVER_OK) + .onSuccess(HttpValueFunctions.jsonContents("ok", Boolean.class))) + .poll(new HttpPollConfig<String>(SERVER_BUILD_TIMESTAMP) + .onSuccess(HttpValueFunctions.jsonContents(new String[]{"version", "build_timestamp"}, String.class))) + .poll(new HttpPollConfig<String>(SERVER_BUILD_HASH) + .onSuccess(HttpValueFunctions.jsonContents(new String[]{"version", "build_hash"}, String.class))) + .poll(new HttpPollConfig<Boolean>(SERVER_IS_BUILD_SNAPSHOT) + .onSuccess(HttpValueFunctions.jsonContents(new String[] {"version", "build_snapshot"}, Boolean.class))) + .poll(new HttpPollConfig<String>(SERVER_LUCENE_VERSION) + .onSuccess(HttpValueFunctions.jsonContents(new String[] {"version", "lucene_version"}, String.class))) + .poll(new HttpPollConfig<String>(SERVER_ES_VERSION) + .onSuccess(HttpValueFunctions.jsonContents(new String[] {"version", "es_version"}, String.class))) .build(); + + addEnricher(Enrichers.builder().updatingMap(Attributes.SERVICE_NOT_UP_INDICATORS) + .from(SERVER_OK) + .computing(Functionals.ifNotEquals(true).value("Crate server reports it is not ok.")) + .build()); } @Override - protected void postStart() { - super.postStart(); + protected void disconnectSensors() { + disconnectServiceUpIsRunning(); + if (jmxFeed != null) jmxFeed.stop(); + if (httpFeed != null) httpFeed.stop(); + super.disconnectSensors(); + } + public Integer getPort() { + return getAttribute(CRATE_PORT); } - private JmxFeed getJmxFeed() { - return JavaAppUtils.connectMXBeanSensors(this); + + public Integer getHttpPort() { + return getAttribute(CRATE_HTTP_PORT); } + } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeSshDriver.java ---------------------------------------------------------------------- diff --git a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeSshDriver.java b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeSshDriver.java index 17ec637..a807cbe 100644 --- a/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeSshDriver.java +++ b/software/database/src/main/java/brooklyn/entity/database/crate/CrateNodeSshDriver.java @@ -1,16 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package brooklyn.entity.database.crate; import static java.lang.String.format; import java.util.List; -import brooklyn.util.collections.MutableMap; import com.google.common.collect.ImmutableList; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.EntityLocal; import brooklyn.entity.java.JavaSoftwareProcessSshDriver; import brooklyn.location.basic.SshMachineLocation; +import brooklyn.util.collections.MutableMap; +import brooklyn.util.net.Urls; import brooklyn.util.os.Os; import brooklyn.util.ssh.BashCommands; @@ -34,7 +53,7 @@ public class CrateNodeSshDriver extends JavaSoftwareProcessSshDriver { List<String> commands = ImmutableList.<String>builder() .addAll(BashCommands.commandsToDownloadUrlsAs(urls, saveAs)) - .add ("tar xvfz "+saveAs) + .add("tar xvfz "+saveAs) .build(); newScript(INSTALLING) @@ -44,16 +63,19 @@ public class CrateNodeSshDriver extends JavaSoftwareProcessSshDriver { @Override public void customize() { - + newScript(CUSTOMIZING) + .body.append("mkdir -p " + getDataLocation()) + .execute(); + copyTemplate(entity.getConfig(CrateNode.SERVER_CONFIG_URL), getConfigFileLocation()); } @Override public void launch() { StringBuilder command = new StringBuilder(getExpandedInstallDir()) - .append("/bin/crate >").append(getLogFileLocation()) - .append(" 2> err.log < /dev/null") + .append("/bin/crate ") .append(" -d") - .append(" -p ").append(getPidFileLocation()); + .append(" -p ").append(getPidFileLocation()) + .append(" -Des.config=").append(getConfigFileLocation()); newScript(LAUNCHING) .failOnNonZeroResultCode() .body.append(command).execute(); @@ -68,17 +90,28 @@ public class CrateNodeSshDriver extends JavaSoftwareProcessSshDriver { @Override public void stop() { - newScript (MutableMap.of("usePidFile", getPidFileLocation()), STOPPING) + // See https://crate.io/docs/stable/cli.html#signal-handling. + newScript(STOPPING) + .body.append("kill -USR2 `cat " + getPidFileLocation() + "`") .execute(); + } + protected String getConfigFileLocation() { + return Urls.mergePaths(getRunDir(), "config.yaml"); } @Override - protected String getLogFileLocation() { - return getRunDir() + "/server.log"; + public String getLogFileLocation() { + return Urls.mergePaths(getRunDir(), "crate.log"); } protected String getPidFileLocation () { - return getRunDir() + "/pid.txt"; + return Urls.mergePaths(getRunDir(), "pid.txt"); + } + + // public for use in template too. + public String getDataLocation() { + return Urls.mergePaths(getRunDir(), "data"); } + } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/main/resources/brooklyn/entity/database/crate/crate.yaml ---------------------------------------------------------------------- diff --git a/software/database/src/main/resources/brooklyn/entity/database/crate/crate.yaml b/software/database/src/main/resources/brooklyn/entity/database/crate/crate.yaml new file mode 100644 index 0000000..42fcee5 --- /dev/null +++ b/software/database/src/main/resources/brooklyn/entity/database/crate/crate.yaml @@ -0,0 +1,28 @@ +# The Crate distribution comes with comprehensive instructions on available +# configuration. Select sections are reproduced here. + + +############################## Network And HTTP ############################### + +# Crate, by default, binds itself to the 0.0.0.0 address, and listens +# on port [4200-4300] for HTTP traffic and on port [4300-4400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set both 'bind_host' and 'publish_host': +network.host: ${driver.subnetHostname} + +# Set a custom port for the node to node communication (4300 by default): +transport.tcp.port: ${entity.port?c} + +# Set a custom port to listen for HTTP traffic: +http.port: ${entity.httpPort?c} + + +#################################### Paths #################################### + +# Path to directory where to store table data allocated for this node. +path.data: ${driver.dataLocation} + +# Path to log files: +path.logs: ${driver.runDir} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/743fd2d0/software/database/src/test/java/brooklyn/entity/database/crate/CrateNodeIntegrationTest.java ---------------------------------------------------------------------- diff --git a/software/database/src/test/java/brooklyn/entity/database/crate/CrateNodeIntegrationTest.java b/software/database/src/test/java/brooklyn/entity/database/crate/CrateNodeIntegrationTest.java new file mode 100644 index 0000000..839fd3f --- /dev/null +++ b/software/database/src/test/java/brooklyn/entity/database/crate/CrateNodeIntegrationTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package brooklyn.entity.database.crate; + +import static org.testng.Assert.assertFalse; + +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; + +import brooklyn.entity.basic.ApplicationBuilder; +import brooklyn.entity.basic.Entities; +import brooklyn.entity.proxying.EntitySpec; +import brooklyn.entity.trait.Startable; +import brooklyn.location.basic.LocalhostMachineProvisioningLocation; +import brooklyn.test.EntityTestUtils; +import brooklyn.test.entity.TestApplication; + +public class CrateNodeIntegrationTest { + + private TestApplication app; + private LocalhostMachineProvisioningLocation localhostProvisioningLocation; + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + localhostProvisioningLocation = new LocalhostMachineProvisioningLocation(); + app = ApplicationBuilder.newManagedApp(TestApplication.class); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() throws Exception { + if (app != null) Entities.destroyAll(app.getManagementContext()); + } + + @Test(groups = "Integration") + public void testCanStartAndStop() throws Exception { + CrateNode entity = app.createAndManageChild(EntitySpec.create(CrateNode.class)); + app.start(ImmutableList.of(localhostProvisioningLocation)); + + EntityTestUtils.assertAttributeEqualsEventually(entity, Startable.SERVICE_UP, true); + EntityTestUtils.assertAttributeEventuallyNonNull(entity, CrateNode.SERVER_NAME); + + entity.stop(); + assertFalse(entity.getAttribute(Startable.SERVICE_UP)); + } +}
