fgerlits commented on code in PR #2066:
URL: https://github.com/apache/nifi-minifi-cpp/pull/2066#discussion_r2598055525
##########
behave_framework/src/minifi_test_framework/core/helpers.py:
##########
@@ -82,3 +83,16 @@ def run_cmd_in_docker_image(image_name: str, cmd: str |
list, network: str) -> s
def run_shell_cmd_in_docker_image(image_name: str, cmd: str, network: str) ->
str:
return run_cmd_in_docker_image(image_name, ["/bin/sh", "-c", cmd], network)
+
+
+def retry_check(max_tries=5, retry_interval=1):
+ def retry_check_func(func):
+ @functools.wraps(func)
+ def retry_wrapper(*args, **kwargs):
+ for _ in range(max_tries):
+ if func(*args, **kwargs):
+ return True
+ time.sleep(retry_interval)
+ return False
+ return retry_wrapper
+ return retry_check_func
Review Comment:
Minor, and I know this was like this before, but it's not great that we do
an extra sleep after the last failure. I think we should add the few extra
lines of code to avoid this.
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in
commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest",
["python", "-c", command], remove=True, stdout=True, stderr=True,
network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase
helper docker: {e}")
+ return False
+
+ @retry_check(max_tries=15, retry_interval=2)
+ def _load_couchbase_certs(self):
+ python_command = f"""
+import requests
+import sys
+from requests.auth import HTTPBasicAuth
+response =
requests.post(f"http://{self.container_name}:8091/node/controller/loadTrustedCAs";,
auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+
+response =
requests.post(f"http://{self.container_name}:8091/node/controller/reloadCertificate";,
auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+sys.exit(0)
+ """
+ return self._run_python_in_couchbase_helper_docker(python_command)
+
+ def is_data_present_in_couchbase(self, doc_id: str, bucket_name: str,
expected_data: str, expected_data_type: str):
+ python_command = f"""
+from couchbase.cluster import Cluster
+from couchbase.options import ClusterOptions
+from couchbase.auth import PasswordAuthenticator
+from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder
+import json
+import sys
+
+try:
+ cluster = Cluster("couchbase://{self.container_name}", ClusterOptions(
+ PasswordAuthenticator('Administrator', 'password123')))
+
+ bucket = cluster.bucket("{bucket_name}")
+ collection = bucket.default_collection()
+
+ if "{expected_data_type}".lower() == "binary":
+ binary_flag = 0x03 << 24
+ result = collection.get("{doc_id}", transcoder=RawBinaryTranscoder())
+ flags = result.flags
+ if not flags & binary_flag:
+ sys.exit(1)
+
+ content = result.content_as[bytes]
+ if content.decode('utf-8') == '{expected_data}':
+ sys.exit(0)
+
+ if "{expected_data_type}".lower() == "json":
+ json_flag = 0x02 << 24
+ result = collection.get("{doc_id}")
+ flags = result.flags
+ if not flags & json_flag:
+ sys.exit(1)
+
+ content = result.content_as[dict]
+ if content == json.loads('{expected_data}'):
+ sys.exit(0)
+ if "{expected_data_type}".lower() == "string":
+ string_flag = 0x04 << 24
+ result = collection.get("{doc_id}", transcoder=RawStringTranscoder())
+ flags = result.flags
+ if not flags & string_flag:
+ print("FAILURE")
Review Comment:
I would keep the old diagnostic messages, e.g.:
https://github.com/apache/nifi-minifi-cpp/blob/ef3dd5656957b39a39edc0bb7f3f426ccc33a1e1/docker/test/integration/cluster/checkers/CouchbaseChecker.py#L59
(and log them in `_run_python_in_couchbase_helper_docker`)
##########
extensions/couchbase/tests/features/couchbase.feature:
##########
@@ -220,28 +248,33 @@ Feature: Executing Couchbase operations from MiNiFi-C++
Scenario: A MiNiFi instance can get data from test bucket with
GetCouchbaseKey processor using mTLS authentication
Given a MiNiFi CPP server with yaml config
Review Comment:
Is there any significance to using YAML config in this Scenario? If not,
then I would remove this line.
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
Review Comment:
it could be useful to capture and log the output of the command, too
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in
commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest",
["python", "-c", command], remove=True, stdout=True, stderr=True,
network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase
helper docker: {e}")
Review Comment:
can we log the stdout and stderr output of command, too? (only if there is a
failure)
##########
extensions/couchbase/tests/features/steps/steps.py:
##########
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from behave import step, then
+
+from minifi_test_framework.steps import checking_steps # noqa: F401
+from minifi_test_framework.steps import configuration_steps # noqa: F401
+from minifi_test_framework.steps import core_steps # noqa: F401
+from minifi_test_framework.steps import flow_building_steps # noqa: F401
+from minifi_test_framework.steps.flow_building_steps import
add_ssl_context_service_for_minifi
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.core.helpers import log_due_to_failure
+from minifi_test_framework.minifi.controller_service import ControllerService
+from couchbase_server_container import CouchbaseServerContainer
+
+
+@step("a Couchbase server is started")
+def step_impl(context: MinifiTestContext):
+ context.containers["couchbase-server"] = CouchbaseServerContainer(context)
+ assert context.containers["couchbase-server"].deploy()
+
+
+@step("a CouchbaseClusterService controller service is set up to communicate
with the Couchbase server")
+def step_impl(context: MinifiTestContext):
+ controller_service =
ControllerService(class_name="CouchbaseClusterService",
service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String",
f"couchbase://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+
+
+@step("a CouchbaseClusterService is set up using SSL connection")
+def step_impl(context):
+ controller_service = ControllerService(class_name="SSLContextService",
service_name="SSLContextService")
+ controller_service.add_property("CA Certificate",
"/tmp/resources/root_ca.crt")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+ controller_service =
ControllerService(class_name="CouchbaseClusterService",
service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String",
f"couchbases://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+ controller_service.add_property("Linked Services", "SSLContextService")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
Review Comment:
please give the two controller services two different names
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
