lordgamez commented on code in PR #2066:
URL: https://github.com/apache/nifi-minifi-cpp/pull/2066#discussion_r2664778224
##########
behave_framework/src/minifi_test_framework/core/helpers.py:
##########
@@ -82,3 +83,16 @@ def run_cmd_in_docker_image(image_name: str, cmd: str |
list, network: str) -> s
def run_shell_cmd_in_docker_image(image_name: str, cmd: str, network: str) ->
str:
return run_cmd_in_docker_image(image_name, ["/bin/sh", "-c", cmd], network)
+
+
+def retry_check(max_tries=5, retry_interval=1):
+ def retry_check_func(func):
+ @functools.wraps(func)
+ def retry_wrapper(*args, **kwargs):
+ for _ in range(max_tries):
+ if func(*args, **kwargs):
+ return True
+ time.sleep(retry_interval)
+ return False
+ return retry_wrapper
+ return retry_check_func
Review Comment:
Updated in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/couchbase.feature:
##########
@@ -220,28 +248,33 @@ Feature: Executing Couchbase operations from MiNiFi-C++
Scenario: A MiNiFi instance can get data from test bucket with
GetCouchbaseKey processor using mTLS authentication
Given a MiNiFi CPP server with yaml config
Review Comment:
I don't see why it is significant here, so I removed it in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92.
Currently all modular docker tests use YAML configuration and the JSON support
will only be introduced in a later PR. After that we can check where it is
worth running tests with both JSON and YAML configurations.
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
Review Comment:
Added in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in
commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest",
["python", "-c", command], remove=True, stdout=True, stderr=True,
network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase
helper docker: {e}")
Review Comment:
Yes it is possible in some cases, added in case of ContainerError in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5",
f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name,
test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM,
cert=test_context.root_ca_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt",
root_ca_content, permissions=0o666))
+ couchbase_cert_content =
crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem",
couchbase_cert_content, permissions=0o666))
+ couchbase_key_content =
crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key",
couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost",
"--cluster-username", "Administrator", "--cluster-password", "password123",
"--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost",
"--username", "Administrator", "--password", "password123", "--bucket",
"test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u",
"Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser",
"--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]",
"--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable",
"prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u',
'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error
code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in
commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest",
["python", "-c", command], remove=True, stdout=True, stderr=True,
network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase
helper docker: {e}")
+ return False
+
+ @retry_check(max_tries=15, retry_interval=2)
+ def _load_couchbase_certs(self):
+ python_command = f"""
+import requests
+import sys
+from requests.auth import HTTPBasicAuth
+response =
requests.post(f"http://{self.container_name}:8091/node/controller/loadTrustedCAs";,
auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+
+response =
requests.post(f"http://{self.container_name}:8091/node/controller/reloadCertificate";,
auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+sys.exit(0)
+ """
+ return self._run_python_in_couchbase_helper_docker(python_command)
+
+ def is_data_present_in_couchbase(self, doc_id: str, bucket_name: str,
expected_data: str, expected_data_type: str):
+ python_command = f"""
+from couchbase.cluster import Cluster
+from couchbase.options import ClusterOptions
+from couchbase.auth import PasswordAuthenticator
+from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder
+import json
+import sys
+
+try:
+ cluster = Cluster("couchbase://{self.container_name}", ClusterOptions(
+ PasswordAuthenticator('Administrator', 'password123')))
+
+ bucket = cluster.bucket("{bucket_name}")
+ collection = bucket.default_collection()
+
+ if "{expected_data_type}".lower() == "binary":
+ binary_flag = 0x03 << 24
+ result = collection.get("{doc_id}", transcoder=RawBinaryTranscoder())
+ flags = result.flags
+ if not flags & binary_flag:
+ sys.exit(1)
+
+ content = result.content_as[bytes]
+ if content.decode('utf-8') == '{expected_data}':
+ sys.exit(0)
+
+ if "{expected_data_type}".lower() == "json":
+ json_flag = 0x02 << 24
+ result = collection.get("{doc_id}")
+ flags = result.flags
+ if not flags & json_flag:
+ sys.exit(1)
+
+ content = result.content_as[dict]
+ if content == json.loads('{expected_data}'):
+ sys.exit(0)
+ if "{expected_data_type}".lower() == "string":
+ string_flag = 0x04 << 24
+ result = collection.get("{doc_id}", transcoder=RawStringTranscoder())
+ flags = result.flags
+ if not flags & string_flag:
+ print("FAILURE")
Review Comment:
Good point, added in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
which should be logged in case of ContainerError
##########
extensions/couchbase/tests/features/steps/steps.py:
##########
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from behave import step, then
+
+from minifi_test_framework.steps import checking_steps # noqa: F401
+from minifi_test_framework.steps import configuration_steps # noqa: F401
+from minifi_test_framework.steps import core_steps # noqa: F401
+from minifi_test_framework.steps import flow_building_steps # noqa: F401
+from minifi_test_framework.steps.flow_building_steps import
add_ssl_context_service_for_minifi
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.core.helpers import log_due_to_failure
+from minifi_test_framework.minifi.controller_service import ControllerService
+from couchbase_server_container import CouchbaseServerContainer
+
+
+@step("a Couchbase server is started")
+def step_impl(context: MinifiTestContext):
+ context.containers["couchbase-server"] = CouchbaseServerContainer(context)
+ assert context.containers["couchbase-server"].deploy()
+
+
+@step("a CouchbaseClusterService controller service is set up to communicate
with the Couchbase server")
+def step_impl(context: MinifiTestContext):
+ controller_service =
ControllerService(class_name="CouchbaseClusterService",
service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String",
f"couchbase://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+
+
+@step("a CouchbaseClusterService is set up using SSL connection")
+def step_impl(context):
+ controller_service = ControllerService(class_name="SSLContextService",
service_name="SSLContextService")
+ controller_service.add_property("CA Certificate",
"/tmp/resources/root_ca.crt")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+ controller_service =
ControllerService(class_name="CouchbaseClusterService",
service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String",
f"couchbases://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+ controller_service.add_property("Linked Services", "SSLContextService")
+
context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
Review Comment:
Good catch, renamed in
https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
