This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 28dc058935b0237a1b7af5380cae47751d9bdf98 Author: Gabor Gyimesi <[email protected]> AuthorDate: Wed Aug 13 15:13:23 2025 +0200 MINIFICPP-2606 Improve logging for python virtualenv initialization Signed-off-by: Gabor Gyimesi <[email protected]> This closes #2008 --- extensions/python/PythonDependencyInstaller.cpp | 65 +++++++++++++++++++--- .../utils/dependency_installer.py | 10 +++- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/extensions/python/PythonDependencyInstaller.cpp b/extensions/python/PythonDependencyInstaller.cpp index 2f2b8854a..1de7557a0 100644 --- a/extensions/python/PythonDependencyInstaller.cpp +++ b/extensions/python/PythonDependencyInstaller.cpp @@ -17,11 +17,14 @@ */ #include "PythonDependencyInstaller.h" +#include <cstdio> + #include "PythonScriptException.h" #include "PythonInterpreter.h" #include "PyException.h" #include "types/Types.h" #include "utils/OptionalUtils.h" +#include "utils/ConfigurationUtils.h" namespace org::apache::nifi::minifi::extensions::python { @@ -49,6 +52,44 @@ std::string encapsulateCommandInQuotesIfNeeded(const std::string& command) { #endif } +#ifdef WIN32 +#define popen _popen +#define pclose _pclose +#endif + +struct CommandResult { + int exit_code; + std::string output; +}; + +CommandResult executeProcess(const std::string& command) { + std::array<char, utils::configuration::DEFAULT_BUFFER_SIZE> buffer{}; + + FILE* pipe = popen(encapsulateCommandInQuotesIfNeeded(command).c_str(), "r"); + if (!pipe) { + return {1, fmt::format("Failed to open pipe for command: {}", command)}; + } + + std::ostringstream result; + while (fgets(buffer.data(), gsl::narrow<int>(buffer.size()), pipe) != nullptr) { + result << buffer.data(); + } + + int status = pclose(pipe); +#ifdef WIN32 + int exit_code = status; +#else + int exit_code = -1; + if (WIFEXITED(status)) { + exit_code = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_code = -WTERMSIG(status); + } +#endif + + return {exit_code, result.str()}; +} + } // namespace PythonDependencyInstaller::PythonDependencyInstaller(const std::shared_ptr<Configure> &configuration) { @@ -93,10 +134,11 @@ void PythonDependencyInstaller::createVirtualEnvIfSpecified() const { } if (!std::filesystem::exists(virtualenv_path_) || std::filesystem::is_empty(virtualenv_path_)) { logger_->log_info("Creating python virtual env at: {}", virtualenv_path_.string()); - auto venv_command = "\"" + python_binary_ + "\" -m venv \"" + virtualenv_path_.string() + "\""; - auto return_value = std::system(encapsulateCommandInQuotesIfNeeded(venv_command).c_str()); - if (return_value != 0) { - throw PythonScriptException(fmt::format("The following command creating python virtual env failed: '{}'", venv_command)); + auto venv_command = "\"" + python_binary_ + "\" -m venv \"" + virtualenv_path_.string() + "\" 2>&1"; + auto result = executeProcess(venv_command); + if (result.exit_code != 0) { + logger_->log_error("The following command creating python virtual env failed: '{}'\nSetup process output:\n{}", venv_command, result.output); + throw PythonScriptException(fmt::format("The following command creating python virtual env failed: '{}'\nSetup process output:\n{}", venv_command, result.output)); } } } @@ -104,14 +146,19 @@ void PythonDependencyInstaller::createVirtualEnvIfSpecified() const { void PythonDependencyInstaller::runInstallCommandInVirtualenv(const std::string& install_command) const { std::string command_with_virtualenv; #if WIN32 - command_with_virtualenv.append("\"").append((virtualenv_path_ / "Scripts" / "activate.bat").string()).append("\" && "); + command_with_virtualenv.append("\"").append((virtualenv_path_ / "Scripts" / "activate.bat").string()).append("\" 2>&1 && "); #else - command_with_virtualenv.append(". \"").append((virtualenv_path_ / "bin" / "activate").string()).append("\" && "); + command_with_virtualenv.append(". \"").append((virtualenv_path_ / "bin" / "activate").string()).append("\" 2>&1 && "); #endif command_with_virtualenv.append(install_command); - auto return_value = std::system(encapsulateCommandInQuotesIfNeeded(command_with_virtualenv).c_str()); - if (return_value != 0) { - throw PythonScriptException(fmt::format("The following command to install python packages failed: '{}'", command_with_virtualenv)); + command_with_virtualenv.append(" 2>&1"); + + auto result = executeProcess(command_with_virtualenv); + if (result.exit_code != 0) { + logger_->log_error("Failed to install python packages to virtualenv with command: {}\nInstall process output:\n{}", command_with_virtualenv, result.output); + throw PythonScriptException(fmt::format("Failed to install python packages to virtualenv with command: {}\nInstall process output:\n{}", command_with_virtualenv, result.output)); + } else { + logger_->log_info("Python packages installed successfully with command: '{}'.\nInstall process output:\n{}", command_with_virtualenv, result.output); } } diff --git a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py index ab0f4b9e4..2a01a23bc 100644 --- a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py +++ b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py @@ -25,7 +25,7 @@ class Visitor(ast.NodeVisitor): for elt in detail.value.elts: # Check if the element is a string constant and add it to the dependencies list if isinstance(elt, ast.Constant): - self.dependencies.append(elt.s) + self.dependencies.append(elt.value) break break @@ -48,7 +48,7 @@ if __name__ == '__main__': print("Installing dependencies for MiNiFi python processors...") # --no-cache-dir is used to be in line with NiFi's dependency install behavior - command = [sys.executable, "-m", "pip", "install", "--no-cache-dir"] + command = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--progress-bar", "off"] dependencies_found = False for i in range(1, len(sys.argv)): if "requirements.txt" in sys.argv[i]: @@ -63,4 +63,8 @@ if __name__ == '__main__': command += dependencies if dependencies_found: - subprocess.check_call(command) + try: + subprocess.check_call(command) + except subprocess.CalledProcessError as e: + print("Error occurred while installing dependencies: {}".format(e)) + sys.exit(1)
