This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 3b2b1b7f790 SOLR-16926: Give override for EmbeddedZK bind host (#1836)
3b2b1b7f790 is described below
commit 3b2b1b7f790986c84671c505e8055afb4dc59c76
Author: Houston Putman <hous...@apache.org>
AuthorDate: Mon Aug 14 11:17:13 2023 -0400
SOLR-16926: Give override for EmbeddedZK bind host (#1836)
---
solr/CHANGES.txt | 4 ++
solr/bin/solr | 4 ++
solr/bin/solr.cmd | 4 ++
solr/bin/solr.in.cmd | 2 +
solr/bin/solr.in.sh | 2 +
.../java/org/apache/solr/cloud/SolrZkServer.java | 41 +++++++------
.../java/org/apache/solr/cloud/ZkController.java | 34 +----------
.../java/org/apache/solr/util/AddressUtils.java | 64 ++++++++++++++++++++
solr/docker/templates/Dockerfile.body.template | 3 +-
.../cases/cloud_multi_node_embedded_zk/test.sh | 70 ++++++++++++++++++++++
.../test.sh | 11 ++--
.../test.sh | 0
solr/server/solr/zoo.cfg | 3 -
.../deployment-guide/pages/securing-solr.adoc | 13 ++++
.../pages/major-changes-in-solr-9.adoc | 4 ++
15 files changed, 195 insertions(+), 64 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9951772eecc..a0ff03bc0bc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -91,6 +91,10 @@ Improvements
* SOLR-14667: Make zkClientTimeout consistent and based on a system property.
The defauls values are stored in a single place referenced everywhere and they
are based on system properties (Alex Deparvu)
+* SOLR-16926: The embedded Zookeeper's bind host can now be overridden, but
still defaults to "127.0.0.1".
+ This is useful when using the ZkCli on a remote Solr using the embedded ZK,
or Solr running in a Docker container.
+ The SOLR_ZK_EMBEDDED_HOST envVar or -Dsolr.zk.embedded.host sysProp control
this bind address. (Houston Putman)
+
Optimizations
---------------------
diff --git a/solr/bin/solr b/solr/bin/solr
index 3ff7832037a..bae4b722432 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1494,6 +1494,10 @@ if [ -n "${SOLR_JETTY_HOST:-}" ]; then
SOLR_OPTS+=("-Dsolr.jetty.host=$SOLR_JETTY_HOST")
fi
+if [ -n "${SOLR_ZK_EMBEDDED_HOST:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.zk.embedded.host=$SOLR_ZK_EMBEDDED_HOST")
+fi
+
: "${STOP_PORT:=$((SOLR_PORT - 1000))}"
if [ "$SCRIPT_CMD" == "start" ] || [ "$SCRIPT_CMD" == "restart" ] ; then
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index 61414f807da..4055bad6d00 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -955,6 +955,10 @@ IF DEFINED SOLR_JETTY_HOST (
set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.jetty.host=%SOLR_JETTY_HOST%"
)
+IF DEFINED SOLR_ZK_EMBEDDED_HOST (
+ set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.zk.embedded.host=%SOLR_ZK_EMBEDDED_HOST%"
+)
+
IF "%SCRIPT_CMD%"=="start" (
REM see if Solr is already running using netstat
For /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^|
find ":%SOLR_PORT% "') do (
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index 0ca0c12c3d5..5f982cb16b5 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -130,6 +130,8 @@ REM set this value as narrowly as required before going to
production. In
REM environments where security is not a concern, 0.0.0.0 can be used to allow
REM Solr to accept connections on all network interfaces.
REM set SOLR_JETTY_HOST=127.0.0.1
+REM Sets the network interface the Embedded ZK binds to.
+REM set SOLR_ZK_EMBEDDED_HOST=127.0.0.1
REM Restrict access to solr by IP address.
REM Specify a comma-separated list of addresses or networks, for example:
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index b1dc6ab5ab4..117ef1761a9 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -154,6 +154,8 @@
# environments where security is not a concern, 0.0.0.0 can be used to allow
# Solr to accept connections on all network interfaces.
#SOLR_JETTY_HOST="127.0.0.1"
+# Sets the network interface the Embedded ZK binds to.
+#SOLR_ZK_EMBEDDED_HOST="127.0.0.1"
# Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this
config
# to enable https module with custom jetty configuration.
diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
index f340f4d6c4d..ee7a14733ed 100644
--- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
@@ -20,9 +20,7 @@ import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.lang.invoke.MethodHandles;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
@@ -30,6 +28,7 @@ import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.common.SolrException;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.AddressUtils;
import org.apache.zookeeper.server.ServerConfig;
import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeer;
@@ -72,7 +71,15 @@ public class SolrZkServer {
if (zkRun == null) return null;
InetSocketAddress addr = zkProps.getClientPortAddress();
- return addr.getHostString() + ":" + addr.getPort();
+ String hostName;
+ // We cannot advertise 0.0.0.0, so choose the best host to advertise
+ // (the same that the Solr Node defaults to)
+ if (addr.getAddress().isAnyLocalAddress()) {
+ hostName = AddressUtils.getHostToAdvertise();
+ } else {
+ hostName = addr.getAddress().getHostAddress();
+ }
+ return hostName + ":" + addr.getPort();
}
public void parseConfig() {
@@ -101,6 +108,10 @@ public class SolrZkServer {
try {
props = SolrZkServerProps.getProperties(zooCfgPath);
SolrZkServerProps.injectServers(props, zkRun, zkHost);
+ // This is the address that the embedded Zookeeper will bind to. Like
Solr, it defaults to
+ // "127.0.0.1".
+ props.setProperty(
+ "clientPortAddress", System.getProperty("solr.zk.embedded.host",
"127.0.0.1"));
if (props.getProperty("clientPort") == null) {
props.setProperty("clientPort", Integer.toString(solrPort + 1000));
}
@@ -145,14 +156,16 @@ public class SolrZkServer {
if (zkProps.getServers().size() > 1) {
if (log.isInfoEnabled()) {
log.info(
- "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}",
- zkProps.getClientPortAddress().getPort());
+ "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening
on host {}",
+ zkProps.getClientPortAddress().getPort(),
+ zkProps.getClientPortAddress().getAddress().getHostAddress());
}
} else {
if (log.isInfoEnabled()) {
log.info(
- "STARTING EMBEDDED STANDALONE ZOOKEEPER SERVER at port {}",
- zkProps.getClientPortAddress().getPort());
+ "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening
on host {}",
+ zkProps.getClientPortAddress().getPort(),
+ zkProps.getClientPortAddress().getAddress().getHostAddress());
}
}
@@ -268,20 +281,6 @@ class SolrZkServerProps extends QuorumPeerConfig {
this.dataDir = dataDir;
}
- public void setClientPort(int clientPort) {
- if (clientPortAddress != null) {
- try {
- this.clientPortAddress =
- new InetSocketAddress(
- InetAddress.getByName(clientPortAddress.getHostName()),
clientPort);
- } catch (UnknownHostException e) {
- throw new RuntimeException(e);
- }
- } else {
- this.clientPortAddress = new InetSocketAddress(clientPort);
- }
- }
-
/**
* Parse config from a Properties.
*
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index c93a7dfef99..25f097cc8c8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -31,15 +31,11 @@ import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Array;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -121,6 +117,7 @@ import
org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.UpdateLog;
+import org.apache.solr.util.AddressUtils;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
import org.apache.zookeeper.CreateMode;
@@ -875,35 +872,8 @@ public class ZkController implements Closeable {
// normalize host removing any url scheme.
// input can be null, host, or url_prefix://host
private String normalizeHostName(String host) {
-
if (host == null || host.length() == 0) {
- String hostaddress;
- try {
- hostaddress = InetAddress.getLocalHost().getHostAddress();
- } catch (UnknownHostException e) {
- hostaddress = "127.0.0.1"; // cannot resolve system hostname, fall
through
- }
- // Re-get the IP again for "127.0.0.1", the other case we trust the hosts
- // file is right.
- if ("127.0.0.1".equals(hostaddress)) {
- Enumeration<NetworkInterface> netInterfaces = null;
- try {
- netInterfaces = NetworkInterface.getNetworkInterfaces();
- while (netInterfaces.hasMoreElements()) {
- NetworkInterface ni = netInterfaces.nextElement();
- Enumeration<InetAddress> ips = ni.getInetAddresses();
- while (ips.hasMoreElements()) {
- InetAddress ip = ips.nextElement();
- if (ip.isSiteLocalAddress()) {
- hostaddress = ip.getHostAddress();
- }
- }
- }
- } catch (Exception e) {
- log.error("Error while looking for a better host name than
127.0.0.1", e);
- }
- }
- host = hostaddress;
+ host = AddressUtils.getHostToAdvertise();
} else {
if (URLUtil.hasScheme(host)) {
host = URLUtil.removeScheme(host);
diff --git a/solr/core/src/java/org/apache/solr/util/AddressUtils.java
b/solr/core/src/java/org/apache/solr/util/AddressUtils.java
new file mode 100644
index 00000000000..ca03e26cf85
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/AddressUtils.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Simple utilities for working Hostname/IP Addresses */
+public final class AddressUtils {
+
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ // normalize host removing any url scheme.
+ // input can be null, host, or url_prefix://host
+ public static String getHostToAdvertise() {
+ String hostaddress;
+ try {
+ hostaddress = InetAddress.getLocalHost().getHostAddress();
+ } catch (UnknownHostException e) {
+ hostaddress =
+ InetAddress.getLoopbackAddress()
+ .getHostAddress(); // cannot resolve system hostname, fall
through
+ }
+ // Re-get the IP again for "127.0.0.1", the other case we trust the hosts
+ // file is right.
+ if ("127.0.0.1".equals(hostaddress)) {
+ try {
+ var netInterfaces = NetworkInterface.getNetworkInterfaces();
+ while (netInterfaces.hasMoreElements()) {
+ NetworkInterface ni = netInterfaces.nextElement();
+ Enumeration<InetAddress> ips = ni.getInetAddresses();
+ while (ips.hasMoreElements()) {
+ InetAddress ip = ips.nextElement();
+ if (ip.isSiteLocalAddress()) {
+ hostaddress = ip.getHostAddress();
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error while looking for a better host name than 127.0.0.1",
e);
+ }
+ }
+ return hostaddress;
+ }
+}
diff --git a/solr/docker/templates/Dockerfile.body.template
b/solr/docker/templates/Dockerfile.body.template
index 7282642a73f..d754b9d7e42 100644
--- a/solr/docker/templates/Dockerfile.body.template
+++ b/solr/docker/templates/Dockerfile.body.template
@@ -45,7 +45,8 @@ ENV SOLR_USER="solr" \
SOLR_PID_DIR=/var/solr \
SOLR_LOGS_DIR=/var/solr/logs \
LOG4J_PROPS=/var/solr/log4j2.xml \
- SOLR_JETTY_HOST="0.0.0.0"
+ SOLR_JETTY_HOST="0.0.0.0" \
+ SOLR_ZK_EMBEDDED_HOST="0.0.0.0"
RUN set -ex; \
groupadd -r --gid "$SOLR_GID" "$SOLR_GROUP"; \
diff --git a/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh
b/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh
new file mode 100755
index 00000000000..2ee47103264
--- /dev/null
+++ b/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+TEST_DIR="${TEST_DIR:-$(dirname -- "${BASH_SOURCE[0]}")}"
+source "${TEST_DIR}/../../shared.sh"
+
+echo "Running base solr node w/embeddedZk - $container_name"
+docker run --name "${container_name}" -d "$tag" solr-fg -c
+
+wait_for_container_and_solr "${container_name}"
+
+solr_ip=$(docker inspect --format="{{range
.NetworkSettings.Networks}}{{.IPAddress}}{{end}}" "${container_name}")
+
+
+container_cleanup "${container_name}-2"
+
+echo "Running additional solr node - $container_name-2"
+docker run --name "$container_name-2" -d \
+ --env "ZK_HOST=${solr_ip}:9983" \
+ "$tag" solr-fg -c
+
+wait_for_container_and_solr "${container_name}-2"
+
+echo "Check live nodes"
+data=$(docker exec --user=solr "${container_name}-2" wget -q -O -
'http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS')
+
+if ! grep -q "${solr_ip}:8983" <<<"$data"; then
+ echo "Test $TEST_NAME $tag failed; could not find first solr node in cluster
state of second node"
+ echo "$data"
+ exit 1
+fi
+
+echo "Creating distributed collection"
+data=$(docker exec --user=solr "$container_name" solr create -c test -rf 1 -s
2)
+
+if ! grep -q "Created collection 'test'" <<<"$data"; then
+ echo "Test $TEST_NAME $tag failed; could not create distributed collection"
+ echo "$data"
+ exit 1
+fi
+
+echo "Submitting Solr query"
+data=$(docker exec --user=solr "${container_name}-2" wget -q -O -
'http://localhost:8983/solr/test/select?q=*:*')
+
+if ! grep -q '"numFound":0' <<<"$data"; then
+ echo "Test $TEST_NAME $tag failed; could not query distributed collection"
+ echo "$data"
+ exit 1
+fi
+
+
+container_cleanup "${container_name}-2"
+container_cleanup "$container_name"
+
+echo "Test $TEST_NAME $tag succeeded"
diff --git a/solr/docker/tests/cases/prometheus-exporter/test.sh
b/solr/docker/tests/cases/prometheus-exporter-cloud/test.sh
similarity index 74%
copy from solr/docker/tests/cases/prometheus-exporter/test.sh
copy to solr/docker/tests/cases/prometheus-exporter-cloud/test.sh
index d964453170b..6407dfeb272 100755
--- a/solr/docker/tests/cases/prometheus-exporter/test.sh
+++ b/solr/docker/tests/cases/prometheus-exporter-cloud/test.sh
@@ -22,27 +22,24 @@ source "${TEST_DIR}/../../shared.sh"
container_cleanup "${container_name}-solr"
echo "Running $container_name"
-docker run --name "${container_name}-solr" -d "$tag" "solr-demo"
+docker run --name "${container_name}-solr" -d "$tag" solr-fg -c
wait_for_container_and_solr "${container_name}-solr"
solr_ip=$(docker inspect --format="{{range
.NetworkSettings.Networks}}{{.IPAddress}}{{end}}" "${container_name}-solr")
-docker run --name "$container_name" --add-host "solr-host:${solr_ip}" -d \
- --env "SOLR_URL=http://solr-host:8983/solr"; \
+docker run --name "$container_name" -d \
+ --env "ZK_HOST=${solr_ip}:9983" \
--env "SCRAPE_INTERVAL=1" \
--env "CLUSTER_ID=myCluster" \
"$tag" "solr-exporter"
wait_for_container_and_solr_exporter "${container_name}"
-echo "Submitting Solr query"
-docker exec --user=solr "${container_name}-solr" wget -q -O -
'http://localhost:8983/solr/demo/select?q=id%3Adell' > /dev/null
-
echo "Checking prometheus data"
data=$(docker exec --user=solr "$container_name" wget -q -O -
'http://localhost:8989/metrics')
-if ! grep -E -q
'solr_metrics_core_query_requests_total{category="QUERY",searchHandler="/select",internal="false",core="demo",base_url="http://solr-host:8983/solr",cluster_id="myCluster",}
[0-9]+.0' <<<"$data"; then
+if ! grep -E -q
"solr_collections_live_nodes{zk_host=\"${solr_ip}:9983\",cluster_id=\"myCluster\",}
1.0" <<<"$data"; then
echo "Test $TEST_NAME $tag failed; did not find correct data"
echo "$data"
exit 1
diff --git a/solr/docker/tests/cases/prometheus-exporter/test.sh
b/solr/docker/tests/cases/prometheus-exporter-standalone/test.sh
similarity index 100%
rename from solr/docker/tests/cases/prometheus-exporter/test.sh
rename to solr/docker/tests/cases/prometheus-exporter-standalone/test.sh
diff --git a/solr/server/solr/zoo.cfg b/solr/server/solr/zoo.cfg
index 4ef8dcea2ae..7c32425e718 100644
--- a/solr/server/solr/zoo.cfg
+++ b/solr/server/solr/zoo.cfg
@@ -11,9 +11,6 @@ syncLimit=5
# dataDir=/opt/zookeeper/data
# NOTE: Solr defaults the dataDir to <solrHome>/zoo_data
-# the address that embedded zookeeper will bind to
-clientPortAddress=127.0.0.1
-
# the port at which the clients will connect
# clientPort=2181
# NOTE: Solr sets this based on zkRun / zkHost params
diff --git
a/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc
b/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc
index 3732cea17cb..9a4fdea3898 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc
@@ -115,4 +115,17 @@ This can be done by setting a `SOLR_JETTY_HOST` value in
your environment's "inc
----
SOLR_JETTY_HOST="0.0.0.0"
----
+
+The same setting is also available as the `-Dsolr.jetty.host` System Property.
+
+The same is true for the embedded Zookeeper, if it is run with Solr.
+By default, the embedded Zookeeper only listens on the loopback interface
("127.0.0.1")
+The bind host is controlled via the `SOLR_ZK_EMBEDDED_HOST` value in your
environment's "include script" (`solr.in.sh` or `solr.in.cmd`):
+
+[source,bash]
+ ----
+ SOLR_ZK_EMBEDDED_HOST="0.0.0.0"
+ ----
+
+The same setting is also available as the `-Dsolr.zk.embedded.host` System
Property.
// end::security-network-binding-1[]
diff --git
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index 750919eabef..2e670ca9f06 100644
---
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -76,6 +76,10 @@ Upgrading existing clouds and use-cases that have custom
configSets will not be
* The default `minimalFreeDiskGB` value has been lowered from `20GB` to `5GB`
when using the
xref:configuration-guide:replica-placement-plugins.adoc#affinityplacementfactory[AffinityPlacementPlugin].
Therefore, when using the default settings, nodes that were previously
excluded from Replica placements due to low available disk space may be
selected after upgrading.
+=== Embedded Zookeeper
+* The Embedded Zookeeper can now be configured to listen to (or bind to) more
hosts than just `localhost`,
+see the
xref:deployment-guide:securing-solr.adoc#network-configuration[Network
Configuration documentation] for more information.
+
== Solr 9.3
=== Binary Releases
