This is an automated email from the ASF dual-hosted git repository.
liuhaopeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new c54034e3 BIGTOP-4408: Download MySQL/Grafana from official website
(#208)
c54034e3 is described below
commit c54034e3d41fde20c7982dbb8499fd985de974b3
Author: Zhiguo Wu <[email protected]>
AuthorDate: Tue Apr 15 14:36:12 2025 +0800
BIGTOP-4408: Download MySQL/Grafana from official website (#208)
---
.../org/apache/bigtop/manager/grpc/pojo/PackageInfo.java | 2 ++
.../server/command/task/AbstractComponentTask.java | 1 +
.../bigtop/manager/server/model/dto/PackageDTO.java | 2 ++
.../bigtop/manager/server/stack/model/PackageModel.java | 2 ++
.../stacks/infra/1.0.0/services/grafana/metainfo.xml | 2 ++
.../stacks/infra/1.0.0/services/mysql/metainfo.xml | 2 ++
.../stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java | 16 +++++++---------
.../bigtop/manager/stack/core/utils/TarballUtils.java | 6 +++++-
.../manager/stack/core/spi/param/BaseParamsTest.java | 4 ++--
9 files changed, 25 insertions(+), 12 deletions(-)
diff --git
a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/PackageInfo.java
b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/PackageInfo.java
index ca8ace38..4dada510 100644
---
a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/PackageInfo.java
+++
b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/PackageInfo.java
@@ -27,6 +27,8 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class PackageInfo {
+ private String url;
+
private String name;
private String checksum;
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
index 9d141fc2..3f50a4a1 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
@@ -127,6 +127,7 @@ public abstract class AbstractComponentTask extends
AbstractTask {
List<PackageInfo> packageInfoList = new ArrayList<>();
for (PackageDTO packageDTO : packageSpecificDTO.getPackages()) {
PackageInfo packageInfo = new PackageInfo();
+ packageInfo.setUrl(packageDTO.getUrl());
packageInfo.setName(packageDTO.getName());
packageInfo.setChecksum(packageDTO.getChecksum());
packageInfoList.add(packageInfo);
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java
index 8cabb51a..eb0c37ab 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java
@@ -27,6 +27,8 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class PackageDTO {
+ private String url;
+
private String name;
private String checksum;
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java
index ca9b2bea..df9a8f45 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java
@@ -27,6 +27,8 @@ import jakarta.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class PackageModel {
+ private String url;
+
private String name;
private String checksum;
diff --git
a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/grafana/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/grafana/metainfo.xml
index 45465e02..87dda446 100644
---
a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/grafana/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/grafana/metainfo.xml
@@ -54,6 +54,7 @@
</architectures>
<packages>
<package>
+ <url>https://dl.grafana.com/oss/release/</url>
<name>grafana-11.4.0.linux-amd64.tar.gz</name>
<checksum>SHA-256:3550c73f4455435642976e82cc89aa354f076a75b766a408781107f4f5d4744c</checksum>
</package>
@@ -65,6 +66,7 @@
</architectures>
<packages>
<package>
+ <url>https://dl.grafana.com/oss/release/</url>
<name>grafana-11.4.0.linux-arm64.tar.gz</name>
<checksum>SHA-256:c978b46a61d92883119131641c03b8a1323a284e74ab9a20e7e48207dc1a11e1</checksum>
</package>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml
index 302df203..00c8e479 100644
---
a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml
@@ -53,6 +53,7 @@
</architectures>
<packages>
<package>
+
<url>https://dev.mysql.com/get/Downloads/MySQL-8.0/</url>
<name>mysql-8.0.40-linux-glibc2.28-x86_64.tar.xz</name>
<checksum>MD5:dcf2702f953d1969be44083f4f063f18</checksum>
</package>
@@ -64,6 +65,7 @@
</architectures>
<packages>
<package>
+
<url>https://dev.mysql.com/get/Downloads/MySQL-8.0/</url>
<name>mysql-8.0.40-linux-glibc2.28-aarch64.tar.xz</name>
<checksum>MD5:a79f41ce62784a1a0e081c76116008de</checksum>
</package>
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java
index 30253fca..373e5dc5 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java
@@ -21,8 +21,6 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hive;
import org.apache.bigtop.manager.common.constants.Constants;
import org.apache.bigtop.manager.common.constants.MessageConstants;
import org.apache.bigtop.manager.common.shell.ShellResult;
-import org.apache.bigtop.manager.common.utils.os.OSDetection;
-import org.apache.bigtop.manager.grpc.pojo.RepoInfo;
import org.apache.bigtop.manager.stack.core.exception.StackException;
import org.apache.bigtop.manager.stack.core.spi.param.Params;
import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript;
@@ -115,12 +113,8 @@ public class HiveMetastoreScript extends
AbstractServerScript {
}
private void downloadMySQLJdbcDriver(Params params) {
- RepoInfo repoInfo = LocalSettings.repos().stream()
- .filter(r -> OSDetection.getArch().equals(r.getArch()) &&
r.getType() == 2)
- .findFirst()
- .orElseThrow(() -> new RuntimeException("Cannot find repo for
os: [" + OSDetection.getOS()
- + "] and arch: [" + OSDetection.getArch() + "]"));
- String mysqlDriver = repoInfo.getBaseUrl() +
"/mysql-connector-j-8.0.33.jar";
+ String mysqlDriver =
+
"https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar";;
TarballDownloader.download(mysqlDriver, params.stackHome());
LinuxFileUtils.moveFile(params.stackHome() +
"/mysql-connector-j-8.0.33.jar", params.serviceHome() + "/lib/");
LinuxFileUtils.updateOwner(params.serviceHome() + "/lib",
params.user(), params.group(), true);
@@ -137,12 +131,16 @@ public class HiveMetastoreScript extends
AbstractServerScript {
&& shellResult.getErrMsg().contains("Table '" +
clusterName + "_hive.VERSION' doesn't exist")) {
// init schema
cmd = hiveParams.serviceHome() + "/bin/schematool -initSchema
-dbType mysql";
+ // TODO previous cmd will kill agent application, not sure why
+ // temp solve by running with nohup and wait for 5 seconds to
let it finish
+ cmd = "nohup " + cmd + " > /dev/null 2>&1 &";
shellResult = LinuxOSUtils.sudoExecCmd(cmd, hiveParams.user());
+ Thread.sleep(5000);
if (shellResult.getExitCode() !=
MessageConstants.SUCCESS_CODE) {
throw new StackException(shellResult.getErrMsg());
}
}
- } catch (IOException e) {
+ } catch (Exception e) {
throw new StackException(e);
}
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
index 9086c356..263694e2 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
@@ -22,6 +22,8 @@ import org.apache.bigtop.manager.grpc.pojo.PackageInfo;
import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor;
+import org.apache.commons.lang3.StringUtils;
+
import lombok.extern.slf4j.Slf4j;
import java.io.File;
@@ -38,7 +40,9 @@ public class TarballUtils {
return;
}
- String remoteUrl = repoUrl + File.separator + packageInfo.getName();
+ String remoteUrl = StringUtils.isEmpty(packageInfo.getUrl())
+ ? repoUrl + File.separator + packageInfo.getName()
+ : packageInfo.getUrl() + File.separator +
packageInfo.getName();
File localFile = new File(stackHome + File.separator +
packageInfo.getName());
TarballDownloader.download(remoteUrl, stackHome, packageInfo);
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java
b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java
index eeed94b1..f3dfb8ed 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java
@@ -65,8 +65,8 @@ public class BaseParamsTest {
arch.add("mockArch");
List<PackageInfo> packages = new ArrayList<>();
- packages.add(new PackageInfo("package1", "testChecksum1"));
- packages.add(new PackageInfo("package2", "testChecksum2"));
+ packages.add(new PackageInfo(null, "package1", "testChecksum1"));
+ packages.add(new PackageInfo(null, "package2", "testChecksum2"));
List<PackageSpecificInfo> packageSpecifics = new ArrayList<>();
PackageSpecificInfo packageSpecific = new PackageSpecificInfo();
