This is an automated email from the ASF dual-hosted git repository.
pgj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2d8a33b9f59a71100a485df7285dc83ecc31725c
Author: Gabor Pali <gabor.p...@ibm.com>
AuthorDate: Sat Nov 18 01:05:31 2023 +0100
Add Clouseau to the developer setup
Provide a way to the developers to deploy and launch the Clouseau
search module more easily, therefore making it more accessible.
This can help with running the Search-based Elixir tests as well as
the Mango `text` search tests. This could allow us to catch more
bugs ahead of time and might even inspire more improvements in the
area.
The extension is designed in a way to make it simple to integrate
with the CI — only the Java environment of the proper version needs
to be deployed, everything else could be managed from this
repository. Such as bumping the version of Clouseau or fine-tuning
the configuration parameters.
This is an opt-in feature on two levels. First, one has to tell
`./configure --enable-clouseau` to instantiate Clouseau locally.
This will create the `clouseau` sub-directory that holds the
contents of the Clouseau distribution of the specified version,
which currently defaults to 2.22.0 (the latest). If an older
version is needed, use the `--clouseau-version` flag.
Then the `mango-test` and `elixir-search` targets will try to use
Clouseau automatically just to make it easy to use in the old way.
However, the `dev/run` script will not do the same. It requires
the `--with-clouseau` flag to be passed for that. That is because
the developer may not necessarily want to launch Clouseau ad hoc,
even if it is available.
With this, note that `elixir-search` is added to the `check` target.
In lack of a configured Clouseau instance, it will become a no-op
and a warning is emitted.
---
.devcontainer/devcontainer.json | 10 +--
Makefile | 25 ++++--
Makefile.win | 28 ++++--
README-DEV.rst | 68 ++++++++++++---
build-aux/Jenkinsfile.full | 18 ++++
build-aux/Jenkinsfile.pr | 4 +-
configure | 80 +++++++++++++++++
configure.ps1 | 64 ++++++++++++++
dev/run | 184 +++++++++++++++++++++++++++++++++++++++-
9 files changed, 444 insertions(+), 37 deletions(-)
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 491b2ecb6..eb45307d3 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -5,11 +5,11 @@
"context": "..",
"args": {
// Useful choices include:
- // apache/couchdbci-debian:bullseye-erlang-25.0.2
- // apache/couchdbci-debian:bullseye-erlang-24.3.4.2
- // apache/couchdbci-debian:bullseye-erlang-23.3.4.15
+ // apache/couchdbci-debian:bullseye-erlang-26.1.2
+ // apache/couchdbci-debian:bullseye-erlang-25.3.2.7
+ // apache/couchdbci-debian:bookworm-erlang-24.3.4.14
//
- "COUCHDB_IMAGE":
"apache/couchdbci-debian:bullseye-erlang-24.3.4.10"
+ "COUCHDB_IMAGE":
"apache/couchdbci-debian:bullseye-erlang-24.3.4.14"
}
},
@@ -23,7 +23,7 @@
// Your code will live inside the volume created for the container under
/workspace.
"workspaceMount": "target=/workspaces/couchdb,type=volume",
"workspaceFolder": "/workspaces/couchdb",
- "postCreateCommand": "./configure --enable-nouveau && make",
+ "postCreateCommand": "./configure --enable-nouveau --enable-clouseau &&
make",
"customizations": {
"vscode": {
diff --git a/Makefile b/Makefile
index 83cc1d360..359cd19d6 100644
--- a/Makefile
+++ b/Makefile
@@ -157,6 +157,7 @@ check: all
@$(MAKE) eunit
@$(MAKE) mango-test
@$(MAKE) elixir
+ @$(MAKE) elixir-search
@$(MAKE) weatherreport-test
@$(MAKE) nouveau-test
@@ -263,12 +264,17 @@ elixir: elixir-init devclean
--no-eval 'mix test --trace --include
test/elixir/test/config/suite.elixir --exclude
test/elixir/test/config/skip.elixir $(EXUNIT_OPTS)'
.PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau
instance
+# target: elixir-search - Run search tests, requires a configured Clouseau
instance
elixir-search: export MIX_ENV=integration
elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
@dev/run -n 1 -q -a adm:pass \
+ --with-clouseau \
--locald-config test/config/test-config.ini \
--no-eval 'mix test --trace --include
test/elixir/test/config/search.elixir'
+else
+ @echo "Warning: Clouseau is not enabled, \`elixir-search\` cannot be
run."
+endif
.PHONY: elixir-source-checks
# target: elixir-source-checks - Check source code formatting of Elixir test
files
@@ -313,19 +319,24 @@ list-eunit-suites:
build-test:
@test/build/test-configure.sh
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
.PHONY: mango-test
# target: mango-test - Run Mango tests
mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
mango-test: devclean all
- @cd src/mango && \
- python3 -m venv .venv && \
- .venv/bin/python3 -m pip install -r requirements.txt
- @cd src/mango && \
- ../../dev/run "$(TEST_OPTS)" \
+ @python3 -m venv src/mango/.venv && \
+ src/mango/.venv/bin/python3 -m pip install -r
src/mango/requirements.txt
+ @dev/run \
+ "$(TEST_OPTS)" \
+ "$(_WITH_CLOUSEAU)" \
-n 1 \
--admin=adm:pass \
- 'COUCH_USER=adm COUCH_PASS=pass .venv/bin/python3 -m nose2
$(MANGO_TEST_OPTS)'
+ --no-eval "\
+COUCH_USER=adm COUCH_PASS=pass \
+src/mango/.venv/bin/nose2 -s src/mango/test -c src/mango/unittest.cfg
$(MANGO_TEST_OPTS)"
.PHONY: weatherreport-test
diff --git a/Makefile.win b/Makefile.win
index b0c6253a8..8f214e325 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -143,6 +143,7 @@ check: all
@$(MAKE) eunit
@$(MAKE) mango-test
@$(MAKE) elixir
+ @$(MAKE) elixir-search
@$(MAKE) nouveau-test
ifdef apps
@@ -203,6 +204,8 @@ python-black-update: .venv/bin/black
--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton"
\
build-aux dev\run dev\format_*.py src\mango\test
src\docs\src\conf.py src\docs\ext .
+-include install.mk
+
ifeq ($(with_nouveau), 0)
exclude_nouveau=--exclude nouveau
endif
@@ -239,12 +242,17 @@ elixir: elixir-init devclean
--no-eval 'mix test --trace --include
test\elixir\test\config\suite.elixir --exclude
test\elixir\test\config\skip.elixir $(EXUNIT_OPTS)'
.PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau
instance
+# target: elixir-search - Run search tests, requires a configured Clouseau
instance
elixir-search: export MIX_ENV=integration
elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
@dev\run -n 1 -q -a adm:pass \
+ --with-clouseau \
--locald-config test/config/test-config.ini \
--no-eval 'mix test --trace --include
test/elixir/test/config/search.elixir'
+else
+ @echo "Warning: Clouseau is not enabled, `elixir-search` cannot be run."
+endif
.PHONY: elixir-source-checks
# target: elixir-source-checks - Check source code formatting of Elixir test
files
@@ -278,19 +286,24 @@ list-eunit-apps:
list-eunit-suites:
@powershell -Command 'Get-ChildItem -Path src -Recurse -Filter
"*_tests?.erl" | ForEach-Object { "{0}" -f $$_.BaseName } | Sort'
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
.PHONY: mango-test
# target: mango-test - Run Mango tests
mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
mango-test: devclean all
- @cd src\mango && \
- python.exe -m venv .venv && \
- .venv\Scripts\pip.exe install -r requirements.txt
- @cd src\mango && \
- ..\..\dev\run "$(TEST_OPTS)" \
+ @$(PYTHON) -m venv src\mango\.venv && \
+ src\mango\.venv\Scripts\pip.exe install -r
src\mango\requirements.txt
+ @dev\run \
+ "$(TEST_OPTS)" \
+ "$(_WITH_CLOUSEAU)" \
-n 1 \
--admin=adm:pass \
- "env COUCH_USER=adm COUCH_PASS=pass .venv\Scripts\nose2
$(MANGO_TEST_OPTS)"
+ "\
+env COUCH_USER=adm COUCH_PASS=pass \
+src\mango\.venv\Scripts\nose2 -s src\mango\test -c src\mango\unittest.cfg
$(MANGO_TEST_OPTS)"
################################################################################
@@ -350,7 +363,6 @@ dist: all derived
.PHONY: release
# target: release - Create an Erlang release including CouchDB!
--include install.mk
release: all
@echo 'Installing CouchDB into rel\couchdb\ ...'
-@rmdir /s/q rel\couchdb >NUL 2>&1 || true
diff --git a/README-DEV.rst b/README-DEV.rst
index d698173c1..3ab5c6bf6 100644
--- a/README-DEV.rst
+++ b/README-DEV.rst
@@ -152,9 +152,9 @@ Configure the source by running::
./configure
-If you intend to run the test suites::
+If you intend to run the test suites with Clouseau::
- ./configure -c
+ ./configure --enable-clouseau
If you don't want to build Fauxton or documentation specify
``--disable-fauxton`` and/or ``--disable-docs`` arguments for ``configure`` to
@@ -237,14 +237,8 @@ but it could be done manually via the corresponding
target::
make elixir-search
-Note that this requires a running Clouseau instance with the name
-``clouseau@127.0.0.1``. The easiest way to get it is to clone the
-`cloudant-labs/clouseau <https://github.com/cloudant-labs/clouseau>`_
-repository and launch it run there once all the prerequisites (JDK,
-Scala, and Maven) have been installed successfully, e.g.::
-
- git clone https://github.com/cloudant-labs/clouseau
- mvn -f clouseau/pom.xml scala:run
+Note that this requires Clouseau to be configured for running, see
+above.
Mango Integration Tests
~~~~~~~~~~~~~~~~~~~~~~~
@@ -262,7 +256,7 @@ the implementation. Consult its documentation for more
information.
Tests that rely on text indexes are run only if the ``search`` feature
is reported to be available (i.e. a working Clouseau instance is
-connected), otherwise they will be skipped.
+configured and working), otherwise they will be skipped.
Note that the databases that are created during the tests will be all
removed after each of the suites completed. However, with the help of
@@ -272,6 +266,58 @@ to keep those databases around for further investigation::
MANGO_TESTS_KEEP_DBS=please \
make mango-test MANGO_TEST_OPTS='03-operator-test'
+Running Clouseau
+~~~~~~~~~~~~~~~~
+
+When configured with the ``./configure`` script, the ``./dev/run``
+script is capable of launching Clouseau instances alongside the
+CouchDB nodes and hooking them up. This is what the ``mango-test``
+and ``elixir-search`` targets also use to run their respective test
+suites, and let Clouseau automatically manage them.
+
+Although the ``./configure`` and the ``./dev/run`` scripts try to take
+care of the details of the Clouseau deployment, it is still the
+responsibility of the user to provide a suitable Java environment for
+running. Clouseau can run with JRE 1.7 and 1.8 only. Also, when
+Nouveau is in use, which uses a more recent Java environment, the old
+JDK has to be installed separately and the ``CLOUSEAU_JAVA_HOME``
+environment variable has to be set to point its location.
+
+Fortunately, the ```asdf`` tool <https://asdf-vm.com/>` provides a
+convenient way to install old versions of JDK through its ```java``
+plugin <https://github.com/halcyon/asdf-java>`::
+
+ asdf plugin add java
+
+Then use ``asdf`` to install it::
+
+ asdf install java zulu-jre-8.74.0.17
+
+Finally, use ``asdf`` to set the ``CLOUSEAU_JAVA_HOME`` environment
+variable::
+
+ export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-jre-8.74.0.17)
+
+If the use of ``asdf`` is not an option, `the Zulu site
+<https://cdn.azul.com/zulu/bin/>` could be used directly to get the
+distribution package for the appropriate JRE version. But this is
+just one of the possibilities to access installers for old Java
+environments.
+
+Once both Clouseau and the corresponding Java environment are set,
+they are not put in use automatically. In order to do so, the
+``./dev/run`` script needs to be run with Clouseau enabled as
+follows::
+
+ dev/run --with-clouseau
+
+When a specific Erlang cookie string is set in
+``rel/overlay/etc/vm.args``, the ``--erlang-cookie`` flag could be
+used to configure Clouseau to work with that::
+
+ dev/run --with-clouseau --erlang-cookie=brumbrum
+
+
Static Code Analysis
~~~~~~~~~~~~~~~~~~~~
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index 80b96a47d..d97346a77 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -27,6 +27,7 @@ meta = [
name: 'CentOS 7',
spidermonkey_vsn: '1.8.5',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
],
@@ -34,6 +35,7 @@ meta = [
name: 'CentOS 8',
spidermonkey_vsn: '60',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
],
@@ -41,6 +43,7 @@ meta = [
name: 'Ubuntu 18.04',
spidermonkey_vsn: '1.8.5',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
],
@@ -48,6 +51,7 @@ meta = [
name: 'Ubuntu 20.04',
spidermonkey_vsn: '68',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
],
@@ -55,6 +59,7 @@ meta = [
name: 'Ubuntu 22.04',
spidermonkey_vsn: '91',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}"
],
@@ -62,6 +67,7 @@ meta = [
name: 'Debian 10',
spidermonkey_vsn: '60',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
],
@@ -72,6 +78,7 @@ meta = [
// name: 'Debian 11 ARM',
// spidermonkey_vsn: '78',
// enable_nouveau: true,
+ // enable_clouseau: false,
// image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}",
// node_label: 'arm64v8'
// ],
@@ -80,6 +87,7 @@ meta = [
name: 'Debian 11 POWER',
spidermonkey_vsn: '78',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}",
node_label: 'ppc64le'
],
@@ -98,6 +106,7 @@ meta = [
name: 'Debian 11',
spidermonkey_vsn: '78',
enable_nouveau: true,
+ enable_clouseau: true,
image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
],
@@ -105,18 +114,21 @@ meta = [
name: 'Debian 12',
spidermonkey_vsn: '78',
enable_nouveau: true,
+ enable_clouseau: false,
image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}"
],
'freebsd-x86_64': [
name: 'FreeBSD',
spidermonkey_vsn: '91',
+ enable_clouseau: false,
gnu_make: 'gmake'
],
'freebsd-arm64': [
name: 'FreeBSD',
spidermonkey_vsn: '91',
+ enable_clouseau: false,
gnu_make: 'gmake'
],
@@ -124,6 +136,7 @@ meta = [
name: 'macOS',
spidermonkey_vsn: '91',
enable_nouveau: false,
+ enable_clouseau: false,
gnu_make: 'make'
]
]
@@ -133,6 +146,9 @@ def String configure(config) {
if (config.enable_nouveau) {
result += " --enable-nouveau"
}
+ if (config.enable_clouseau) {
+ result += " --enable-clouseau"
+ }
return result
}
@@ -170,6 +186,7 @@ def generateNativeStage(platform) {
sh '$MAKE'
sh '$MAKE eunit'
sh '$MAKE elixir'
+ sh '$MAKE elixir-search'
sh '$MAKE mango-test'
sh '$MAKE weatherreport-test'
sh '$MAKE nouveau-test'
@@ -223,6 +240,7 @@ def generateContainerStage(platform) {
sh 'make'
sh 'make eunit'
sh 'make elixir'
+ sh 'make elixir-search'
sh 'make mango-test'
sh 'make weatherreport-test'
sh 'make nouveau-test'
diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr
index af629c792..551c069ff 100644
--- a/build-aux/Jenkinsfile.pr
+++ b/build-aux/Jenkinsfile.pr
@@ -20,7 +20,7 @@ mkdir build
cd build
tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
cd apache-couchdb-*
-./configure --enable-nouveau
+./configure --enable-nouveau --enable-clouseau
make check || (make build-report && false)
'''
@@ -218,7 +218,7 @@ pipeline {
steps {
sh '''
rm -rf apache-couchdb-*
- ./configure --spidermonkey-version 78 --enable-nouveau
+ ./configure --spidermonkey-version 78 --enable-nouveau
--enable-clouseau
make dist
chmod -R a+w * .
'''
diff --git a/configure b/configure
index 8ed9817d2..a6032da68 100755
--- a/configure
+++ b/configure
@@ -29,6 +29,7 @@ WITH_PROPER="true"
WITH_FAUXTON=1
WITH_DOCS=1
WITH_NOUVEAU=0
+WITH_CLOUSEAU=0
ERLANG_MD5="false"
SKIP_DEPS=0
@@ -38,6 +39,8 @@ run_erlang() {
COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)"
SM_VSN=${SM_VSN:-"91"}
+CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.22.0"}
+CLOUSEAU_DIR="$(pwd)"/clouseau
ARCH="$(uname -m)"
ERLANG_VER="$(run_erlang 'io:put_chars(erlang:system_info(otp_release)).')"
ERLANG_OS="$(run_erlang 'case os:type() of {OS, _} -> io:format("~s~n", [OS])
end.')"
@@ -59,9 +62,11 @@ Options:
--disable-fauxton do not build Fauxton
--disable-docs do not build any documentation or manpages
--enable-nouveau enable the new experimental search module
+ --enable-clouseau enable the Clouseau search module
--erlang-md5 use erlang for md5 hash operations
--dev alias for --disable-docs --disable-fauxton
--spidermonkey-version VSN specify the version of SpiderMonkey to use
(defaults to $SM_VSN)
+ --clouseau-version VSN specify the version of Clouseau to use (defaults
to $CLOUSEAU_VSN)
--skip-deps do not update erlang dependencies
--rebar=PATH use rebar by specified path (version >=2.6.0 &&
<3.0 required)
--rebar3=PATH use rebar3 by specified path
@@ -101,6 +106,17 @@ parse_opts() {
continue
;;
+ --enable-clouseau)
+ if [ "${ERLANG_VER}" -gt 25 ]; then
+ echo "Warning: Clouseau is not working with OTP 26 or later
at the moment, therefore it is not configured. Continue in 5 seconds..."
+ sleep 5
+ else
+ WITH_CLOUSEAU=1
+ fi
+ shift
+ continue
+ ;;
+
--erlang-md5)
ERLANG_MD5="true"
shift
@@ -202,6 +218,24 @@ parse_opts() {
exit 1
;;
+ --clouseau-version)
+ if [ -n "$2" ]; then
+ eval CLOUSEAU_SVN=$2
+ shift 2
+ continue
+ else
+ printf 'ERROR: "--clouseau-version" requires a non-empty
argument.\n' >&2
+ exit 1
+ fi
+ ;;
+ --clouseau-version=?*)
+ eval CLOUSEAU_VSN=${1#*=}
+ ;;
+ --clouseau-version=)
+ printf 'ERROR: "--clouseau-version" requires a non-empty
argument.\n' >&2
+ exit 1
+ ;;
+
--) # End of options
shift
break
@@ -298,6 +332,7 @@ package_author_name = $PACKAGE_AUTHOR_NAME
with_fauxton = $WITH_FAUXTON
with_docs = $WITH_DOCS
with_nouveau = $WITH_NOUVEAU
+with_clouseau = $WITH_CLOUSEAU
user = $COUCHDB_USER
spidermonkey_version = $SM_VSN
@@ -345,6 +380,45 @@ install_local_erlfmt() {
fi
}
+install_local_clouseau() {
+
_DIST_URL=https://github.com/cloudant-labs/clouseau/releases/download/"$CLOUSEAU_VSN"/clouseau-"$CLOUSEAU_VSN"-dist.zip
+ _MAVEN_BASE_URI=https://repo1.maven.org/maven2
+
+ _SLF4J_SIMPLE_VSN=${SLF4J_SIMPLE_VERSION:-1.7.36}
+ _SLF4J_SIMPLE_JAR=slf4j-simple-"$_SLF4J_SIMPLE_VSN".jar
+
_SLF4J_SIMPLE_URL="$_MAVEN_BASE_URI"/org/slf4j/slf4j-simple/"$_SLF4J_SIMPLE_VSN"/"$_SLF4J_SIMPLE_JAR"
+
+ rm -rf "$CLOUSEAU_DIR"
+ mkdir -p "$CLOUSEAU_DIR"
+
+ if ! curl -sSL --max-redirs 1 -o clouseau.zip "$_DIST_URL"; then
+ printf "ERROR: %s could not be downloaded.\n" "$_DIST_URL" >&2
+ exit 1
+ fi
+
+ if ! unzip -q -j clouseau.zip -d "$CLOUSEAU_DIR"; then
+ printf "ERROR: Clouseau distribution package (clouseau.zip) could not
be extracted.\n" >&2
+ exit 1
+ fi
+
+ rm clouseau.zip
+
+ if ! curl -sSL --max-redirs 1 -o "$CLOUSEAU_DIR"/"$_SLF4J_SIMPLE_JAR"
"$_SLF4J_SIMPLE_URL"; then
+ printf "ERROR: %s could not be downloaded.\n" "$_SLF4J_SIMPLE_URL" >&2
+ exit 1
+ fi
+
+ cat <<EOF > "$CLOUSEAU_DIR"/clouseau.ini
+[clouseau]
+EOF
+ cat <<EOF > "$CLOUSEAU_DIR"/log4j.properties
+log4j.rootLogger=debug, CONSOLE
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n
+EOF
+}
+
if [ -z "${REBAR}" ]; then
install_local_rebar
REBAR=${rootdir}/bin/rebar
@@ -360,6 +434,12 @@ if [ -z "${ERLFMT}" ]; then
ERLFMT=${rootdir}/bin/erlfmt
fi
+if [ $WITH_CLOUSEAU -ne 0 ]; then
+ install_local_clouseau
+else
+ rm -rf "$CLOUSEAU_DIR"
+fi
+
# only update dependencies, when we are not in a release tarball
if [ -d .git -a $SKIP_DEPS -ne 1 ]; then
echo "==> updating dependencies"
diff --git a/configure.ps1 b/configure.ps1
index ec0b650db..7d13b5505 100644
--- a/configure.ps1
+++ b/configure.ps1
@@ -8,9 +8,11 @@
-DisableFauxton request build process skip building Fauxton
(default false)
-DisableDocs request build process skip building documentation
(default false)
-EnableNouveau enable the new experiemtal search module (default
false)
+ -EnableClouseau enable the Clouseau search module (default false)
-SkipDeps do not update Erlang dependencies (default false)
-CouchDBUser USER set the username to run as (defaults to current
user)
-SpiderMonkeyVersion VSN select the version of SpiderMonkey to use
(default 91)
+ -ClouseauVersion VSN select the version of Clouseau to use (default
2.22.0)
Installation directories:
-Prefix PREFIX install architecture-independent files in PREFIX
@@ -45,6 +47,7 @@ Param(
[switch]$DisableFauxton = $false, # do not build Fauxton
[switch]$DisableDocs = $false, # do not build any documentation or manpages
[switch]$EnableNouveau = $false, # dont use new search module by default
+ [switch]$EnableClouseau = $false, # do not use Clouseau by default
[switch]$SkipDeps = $false, # do not update erlang dependencies
[switch]$DisableProper = $false, # a compilation pragma. proper is a kind
of automated test suite
[switch]$EnableErlangMD5 = $false, # don't use Erlang for md5 hash
operations by default
@@ -54,6 +57,8 @@ Param(
[ValidateNotNullOrEmpty()]
[string]$SpiderMonkeyVersion = "91", # select the version of SpiderMonkey
to use (default 91)
[ValidateNotNullOrEmpty()]
+ [string]$ClouseauVersion = "2.22.0", # select the version of Clouseau to
use (default 2.22.0)
+ [ValidateNotNullOrEmpty()]
[string]$Prefix = "C:\Program Files\Apache\CouchDB", # install
architecture-independent file location (default C:\Program Files\Apache\CouchDB)
[ValidateNotNullOrEmpty()]
[string]$ExecPrefix = $Prefix, # install architecture-dependent file
location (default C:\Program Files\Apache\CouchDB)
@@ -130,6 +135,7 @@ $LogFile="$LogDir\couch.log"
$BuildFauxton = [int](-not $DisableFauxton)
$BuildDocs = [int](-not $DisableDocs)
$BuildNouveau = $(If ($EnableNouveau) {1} else {0})
+$WithClouseau = $(If ($EnableClouseau) {1} else {0})
$Hostname = [System.Net.Dns]::GetHostEntry([string]"localhost").HostName
$WithProper = (-not $DisableProper).ToString().ToLower()
$ErlangMD5 = ($EnableErlangMD5).ToString().ToLower()
@@ -201,6 +207,7 @@ man_dir = $ManDir
with_fauxton = $BuildFauxton
with_docs = $BuildDocs
with_nouveau = $BuildNouveau
+with_clouseau = $WithClouseau
user = $CouchDBUser
spidermonkey_version = $SpiderMonkeyVersion
@@ -267,6 +274,63 @@ if ((Get-Command "erlfmt.cmd" -ErrorAction
SilentlyContinue) -eq $null)
cd ..\..
}
+$ClouseauDir = "$rootdir\clouseau"
+
+if ($EnableClouseau)
+{
+ Write-Verbose "===> downloading Clouseau distribution..."
+
+ if (Test-Path $ClouseauDir) {
+ Remove-Item -Recurse -Force $ClouseauDir
+ }
+ New-Item -Path $ClouseauDir -ItemType Directory | Out-File Null
+
+ $Slf4jVersion = "1.7.36"
+ $ClouseauDistUrl =
"https://github.com/cloudant-labs/clouseau/releases/download/$ClouseauVersion/clouseau-$ClouseauVersion-dist.zip";
+ $Slf4jSimpleJar = "slf4j-simple-$Slf4jVersion.jar"
+ $Slf4jSimpleUrl =
"https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/$Slf4jVersion/$Slf4jSimpleJar";
+
+ Set-Variable ProgressPreference SilentlyContinue
+ Invoke-WebRequest -MaximumRedirection 1 -OutFile clouseau.zip
$ClouseauDistUrl
+ If ($LASTEXITCODE -ne 0) {
+ Write-Output "ERROR: $ClouseauDistUrl could not be downloaded."
+ exit 1
+ }
+
+ Expand-Archive clouseau.zip -DestinationPath $ClouseauDir -Force
+ If ($LASTEXITCODE -ne 0) {
+ Write-Output "ERROR: Clouseau distribution package (clouseau.zip) could
not be extracted."
+ exit 1
+ }
+ mv "$ClouseauDir\*\*.jar" "$ClouseauDir"
+ rm "$ClouseauDir\clouseau-$ClouseauVersion"
+ rm clouseau.zip
+
+ Invoke-WebRequest -MaximumRedirection 1 -OutFile
"$ClouseauDir\$Slf4jSimpleJar" $Slf4jSimpleUrl
+ If ($LASTEXITCODE -ne 0) {
+ Write-Output "ERROR: $Slf4jSimpleJarUrl could not be downloaded."
+ exit 1
+ }
+
+ $ClouseauIni = @"
+[clouseau]
+"@
+ $ClouseauIni | Out-File "$ClouseauDir\clouseau.ini" -encoding ascii
+
+ $Log4JProperties = @"
+log4j.rootLogger=debug, CONSOLE
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n
+"@
+ $Log4JProperties | Out-File "$ClouseauDir\log4j.properties"
+}
+else {
+ if (Test-Path $ClouseauDir) {
+ Remove-Item -Recurse -Force $ClouseauDir
+ }
+}
+
# only update dependencies, when we are not in a release tarball
if ( (Test-Path .git -PathType Container) -and (-not $SkipDeps) ) {
Write-Verbose "==> updating dependencies"
diff --git a/dev/run b/dev/run
index 707c8a4d7..7bb807bf2 100755
--- a/dev/run
+++ b/dev/run
@@ -23,6 +23,7 @@ import json
import ntpath
import optparse
import os
+import platform
import posixpath
import re
import signal
@@ -90,11 +91,15 @@ log.verbose = True
def main():
ctx = setup()
- startup(ctx)
- if ctx["cmd"]:
- run_command(ctx, ctx["cmd"])
+ try:
+ startup(ctx)
+ except ClouseauError:
+ sys.exit(1)
else:
- join(ctx, cluster_port(ctx, 1), *ctx["admin"])
+ if ctx["cmd"]:
+ run_command(ctx, ctx["cmd"])
+ else:
+ join(ctx, cluster_port(ctx, 1), *ctx["admin"])
def setup():
@@ -235,6 +240,19 @@ def get_args_parser():
action="store_true",
help="Start Nouveau server",
)
+ parser.add_option(
+ "--with-clouseau",
+ dest="with_clouseau",
+ default=False,
+ action="store_true",
+ help="Start Clouseau nodes",
+ )
+ parser.add_option(
+ "--erlang-cookie",
+ dest="erlang_cookie",
+ default=None,
+ help="Erlang cookie string",
+ )
parser.add_option(
"-t",
"--enable-tls",
@@ -279,6 +297,8 @@ def setup_context(opts, args):
"auto_ports": opts.auto_ports,
"locald_configs": opts.locald_configs,
"with_nouveau": opts.with_nouveau,
+ "with_clouseau": opts.with_clouseau,
+ "erlang_cookie": opts.erlang_cookie,
"enable_tls": opts.enable_tls,
"no_tls": opts.no_tls,
}
@@ -505,6 +525,157 @@ def boot_nouveau(ctx):
)
+CLOUSEAU_DIR = "clouseau"
+JAVA_VERSION_RE = re.compile('"(\d+\.\d+).*"')
+
+
+def get_java_version(java):
+ try:
+ output = sp.check_output([java, "-version"],
stderr=sp.STDOUT).decode("utf-8")
+ except:
+ output = None
+
+ if output:
+ matches = JAVA_VERSION_RE.search(output)
+ if matches:
+ return float(matches.groups()[0])
+
+
+class ClouseauError(Exception):
+ def __init__(self, message):
+ self.message = message
+ super().__init__(self.message)
+
+
+@log("Start Clouseau node clouseau{idx}")
+def boot_clouseau(ctx, idx):
+ configure_cmd = (
+ "&.\\configure.ps1 -EnableClouseau"
+ if platform.system() == "Windows"
+ else "./configure --enable-clouseau"
+ )
+
+ if not os.path.isdir(CLOUSEAU_DIR):
+ raise ClouseauError(
+ "Clouseau deployment cannot be found, please run
`{}`".format(configure_cmd)
+ )
+
+ clouseau_jdk_home = os.environ.get("CLOUSEAU_JAVA_HOME") or os.environ.get(
+ "JAVA_HOME"
+ )
+ java = clouseau_jdk_home + "/bin/java" if clouseau_jdk_home else "java"
+ java_version = get_java_version(java)
+
+ if not java_version:
+ print(
+ "Warning: Java version could not be determined, Clouseau may not
be able to run"
+ )
+ else:
+ if java_version < 1.7 or java_version > 1.8:
+ raise ClouseauError(
+ "Java is not suitable to run Clouseau. Please use JRE 1.7 or
1.8 and configure its (root) path in `CLOUSEAU_JAVA_HOME`"
+ )
+
+ clouseau_jars = [
+ "{}/{}".format(CLOUSEAU_DIR, fname)
+ for fname in os.listdir(CLOUSEAU_DIR)
+ if fname.endswith(".jar")
+ ]
+
+ if not clouseau_jars:
+ raise ClouseauError("Clouseau has no JAR files")
+
+ clouseau_ini = "{}/clouseau.ini".format(CLOUSEAU_DIR)
+ if not os.path.isfile(clouseau_ini):
+ raise ClouseauError("Clouseau has no ini file")
+
+ log4j_properties = "{}/log4j.properties".format(CLOUSEAU_DIR)
+ if not os.path.isfile(log4j_properties):
+ raise ClouseauError("Clouseau has no Log4J configuration")
+
+ clouseau_name = "clouseau{}@127.0.0.1".format(idx)
+ clouseau_indexes_dir = "{}/clouseau{}/data".format(ctx["devdir"], idx)
+
+ if ctx["erlang_cookie"]:
+ clouseau_cookie = ["-Dclouseau.cookie={}".format(ctx["erlang_cookie"])]
+ else:
+ clouseau_cookie = []
+
+ # `HOME` must be set for Scalang to find the Erlang cookie
+ if not os.environ.get("HOME"):
+ # This is usually `USERPROFILE` on Windows
+ alternativeHome = os.environ.get("USERPROFILE")
+ if alternativeHome:
+ os.environ["HOME"] = alternativeHome
+
+ separator = ";" if platform.system() == "Windows" else ":"
+
+ cmd = (
+ [
+ java,
+ "-server",
+ "-Xmx2G",
+ "-Dsun.net.inetaddr.ttl=30",
+ "-Dsun.net.inetaddr.negative.ttl=30",
+ '-XX:OnOutOfMemoryError="kill -9 %p"',
+ "-XX:+UseConcMarkSweepGC",
+ "-XX:+CMSParallelRemarkEnabled",
+ "-cp",
+ separator.join(clouseau_jars),
+ "-Dlog4j.configuration=file:{}".format(log4j_properties),
+ "-Dclouseau.name={}".format(clouseau_name),
+ "-Dclouseau.dir={}".format(clouseau_indexes_dir),
+ ]
+ + clouseau_cookie
+ + [
+ "com.cloudant.clouseau.Main",
+ clouseau_ini,
+ ]
+ )
+ logfname = os.path.join(ctx["devdir"], "logs",
"clouseau{}.log".format(idx))
+ log = open(logfname, "w")
+
+ try:
+ return sp.Popen(
+ cmd,
+ stdin=sp.PIPE,
+ stdout=log,
+ stderr=sp.STDOUT,
+ )
+ except:
+ raise ClouseauError("Java is not capable of running Clouseau")
+
+
+def maybe_boot_clouseau(ctx, idx):
+ if ctx["with_clouseau"]:
+ return boot_clouseau(ctx, idx)
+
+
+@log("Check Clouseau node clouseau{idx}")
+def check_clouseau_node_alive(ctx, idx):
+ if ctx["erlang_cookie"]:
+ cookie = ["-c", ctx["erlang_cookie"]]
+ else:
+ cookie = []
+
+ cmd = (
+ ["erl_call", "-n", "node{}@127.0.0.1".format(idx)]
+ + cookie
+ + [
+ "-a",
+ "net_adm ping ['clouseau{}@127.0.0.1']".format(idx),
+ ]
+ )
+
+ if sp.check_output(cmd) != b"pong":
+ raise ValueError("Clouseau is not working")
+
+
+def maybe_check_clouseau_node_alive(ctx, idx):
+ if ctx["with_clouseau"]:
+ check_clouseau_node_alive(ctx, idx)
+
+
def hack_default_ini(ctx, node, contents):
contents = re.sub(
"^\[httpd\]$",
@@ -628,6 +799,10 @@ def boot_nodes(ctx):
nouveau_proc = boot_nouveau(ctx)
if nouveau_proc is not None:
ctx["procs"].append(nouveau_proc)
+ for idx in [(i + ctx["node_number"]) for i in range(ctx["N"])]:
+ clouseau_proc = maybe_boot_clouseau(ctx, idx)
+ if clouseau_proc is not None:
+ ctx["procs"].append(clouseau_proc)
def ensure_all_nodes_alive(ctx):
@@ -640,6 +815,7 @@ def ensure_all_nodes_alive(ctx):
url = "http://127.0.0.1:{0}/".format(local_port)
try:
check_node_alive(url)
+ maybe_check_clouseau_node_alive(ctx, num + 1)
except:
pass
else:
