This is an automated email from the ASF dual-hosted git repository. wuzhiguo pushed a commit to branch branch-1.0 in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
commit b3ca1078fbd87a802bcc1916e1b9b19ad1ddf4be Author: Zhiguo Wu <[email protected]> AuthorDate: Sun Apr 20 23:29:03 2025 +0800 BIGTOP-4414: Move tool info to database (#214) --- .../grpc/service/SetupJdkServiceGrpcImpl.java | 51 +++------------------- .../org/apache/bigtop/manager/dao/po/RepoPO.java | 3 -- .../manager/dao/po/{RepoPO.java => ToolPO.java} | 17 +++++--- .../bigtop/manager/dao/repository/ToolDao.java | 15 ++----- .../bigtop/manager/grpc/pojo/ClusterInfo.java | 4 ++ .../apache/bigtop/manager/grpc/pojo/RepoInfo.java | 2 - .../grpc/pojo/{ClusterInfo.java => ToolInfo.java} | 10 +++-- .../server/command/helper/JobCacheHelper.java | 19 +++++++- .../bigtop/manager/server/model/dto/RepoDTO.java | 2 - .../server/service/impl/HostServiceImpl.java | 4 +- .../src/main/resources/ddl/MySQL-DDL-CREATE.sql | 34 +++++++++++---- .../main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 36 ++++++++++----- .../stacks/bigtop/3.3.0/services/flink/order.json | 2 +- .../bigtop/v3_3_0/hive/HiveMetastoreScript.java | 10 +++-- .../stack/bigtop/v3_3_0/tez/TezClientScript.java | 4 +- .../manager/stack/core/spi/param/BaseParams.java | 3 +- ...{TarballDownloader.java => FileDownloader.java} | 12 ++++- .../manager/stack/core/utils/LocalSettings.java | 25 ++++++++--- .../manager/stack/core/utils/TarballUtils.java | 4 +- .../stack/core/spi/param/BaseParamsTest.java | 6 +-- .../stack/core/utils/LocalSettingsTest.java | 8 ++-- docs/en/cluster.md | 4 +- docs/zh/cluster.md | 4 +- 23 files changed, 155 insertions(+), 124 deletions(-) diff --git a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/service/SetupJdkServiceGrpcImpl.java b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/service/SetupJdkServiceGrpcImpl.java index bdb0d048..4a999454 100644 --- a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/service/SetupJdkServiceGrpcImpl.java +++ b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/service/SetupJdkServiceGrpcImpl.java @@ -21,13 +21,12 @@ package org.apache.bigtop.manager.agent.grpc.service; import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.constants.MessageConstants; import org.apache.bigtop.manager.common.utils.Environments; -import org.apache.bigtop.manager.common.utils.os.OSDetection; import org.apache.bigtop.manager.grpc.generated.SetupJdkReply; import org.apache.bigtop.manager.grpc.generated.SetupJdkRequest; import org.apache.bigtop.manager.grpc.generated.SetupJdkServiceGrpc; import org.apache.bigtop.manager.grpc.pojo.ClusterInfo; import org.apache.bigtop.manager.grpc.pojo.PackageInfo; -import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.ToolInfo; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.TarballUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; @@ -36,8 +35,6 @@ import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; -import java.text.MessageFormat; - @Slf4j @GrpcService public class SetupJdkServiceGrpcImpl extends SetupJdkServiceGrpc.SetupJdkServiceImplBase { @@ -55,26 +52,18 @@ public class SetupJdkServiceGrpcImpl extends SetupJdkServiceGrpc.SetupJdkService } log.info("Setting up cluster jdk..."); - String arch = OSDetection.getArch(); - String pkgName = getPkgName(arch); - String checksum = getChecksum(arch); - - PackageInfo packageInfo = new PackageInfo(); - packageInfo.setName(pkgName); - packageInfo.setChecksum(checksum); - ClusterInfo clusterInfo = LocalSettings.cluster(); - RepoInfo repoInfo = LocalSettings.repos().stream() - .filter(r -> arch.equals(r.getArch()) && r.getType() == 2) - .findFirst() - .orElseThrow(() -> new RuntimeException( - "Cannot find repo for os: [" + OSDetection.getOS() + "] and arch: [" + arch + "]")); String toolsHome = clusterInfo.getRootDir() + "/tools"; String user = System.getProperty("user.name"); LinuxFileUtils.createDirectories(toolsHome, user, user, Constants.PERMISSION_755, true); String jdkHome = toolsHome + "/jdk"; - TarballUtils.installPackage(repoInfo.getBaseUrl(), toolsHome, jdkHome, packageInfo, 1); + ToolInfo tool = LocalSettings.getTool("jdk8"); + PackageInfo packageInfo = new PackageInfo(); + packageInfo.setUrl(tool.getBaseUrl()); + packageInfo.setName(tool.getPkgName()); + packageInfo.setChecksum(tool.getChecksum()); + TarballUtils.installPackage(null, toolsHome, jdkHome, packageInfo, 1); LinuxFileUtils.createDirectories(jdkHome, user, user, Constants.PERMISSION_755, true); SetupJdkReply reply = SetupJdkReply.newBuilder() @@ -87,30 +76,4 @@ public class SetupJdkServiceGrpcImpl extends SetupJdkServiceGrpc.SetupJdkService responseObserver.onError(e); } } - - private String getPkgName(String arch) { - String replacedArch = - switch (arch) { - case "x86_64" -> "x64"; - case "arm64", "aarch64" -> "aarch64"; - default -> { - log.error("Unsupported architecture: {}", arch); - throw new IllegalArgumentException("Unsupported architecture: " + arch); - } - }; - String pkgName = MessageFormat.format("jdk-8u431-linux-{0}.tar.gz", replacedArch); - log.debug("Generated package name: {}", pkgName); - return pkgName; - } - - private String getChecksum(String arch) { - return switch (arch) { - case "x64", "x86_64" -> "SHA-256:b396978a716b7d23ccccabfe5c47c3b75d2434d7f8f7af690bc648172382720d"; - case "arm64", "aarch64" -> "SHA-256:e68d3e31ffcf7f05a4de65d04974843073bdff238bb6524adb272de9e616be7c"; - default -> { - log.error("Unknown arch for jdk: {}", arch); - throw new IllegalArgumentException("Unknown arch for jdk: " + arch); - } - }; - } } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java index a4e0ec8f..bd64fc49 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java @@ -43,7 +43,4 @@ public class RepoPO extends BasePO implements Serializable { @Column(name = "base_url") private String baseUrl; - - @Column(name = "type") - private Integer type; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ToolPO.java similarity index 86% copy from bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java copy to bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ToolPO.java index a4e0ec8f..b7e16ca9 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ToolPO.java @@ -28,8 +28,8 @@ import java.io.Serializable; @Data @EqualsAndHashCode(callSuper = true) -@Table(name = "repo") -public class RepoPO extends BasePO implements Serializable { +@Table(name = "tool") +public class ToolPO extends BasePO implements Serializable { @Id @Column(name = "id") @@ -38,12 +38,15 @@ public class RepoPO extends BasePO implements Serializable { @Column(name = "name") private String name; - @Column(name = "arch") - private String arch; - @Column(name = "base_url") private String baseUrl; - @Column(name = "type") - private Integer type; + @Column(name = "pkg_name") + private String pkgName; + + @Column(name = "arch") + private String arch; + + @Column(name = "checksum") + private String checksum; } diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ToolDao.java similarity index 76% copy from bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java copy to bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ToolDao.java index 957e0c86..3306e41c 100644 --- a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ToolDao.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,16 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.grpc.pojo; -import lombok.Data; +package org.apache.bigtop.manager.dao.repository; -@Data -public class ClusterInfo { +import org.apache.bigtop.manager.dao.po.ToolPO; - private String name; - - private String userGroup; - - private String rootDir; -} +public interface ToolDao extends BaseDao<ToolPO> {} diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java index 957e0c86..358e646a 100644 --- a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java +++ b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java @@ -20,6 +20,8 @@ package org.apache.bigtop.manager.grpc.pojo; import lombok.Data; +import java.util.List; + @Data public class ClusterInfo { @@ -28,4 +30,6 @@ public class ClusterInfo { private String userGroup; private String rootDir; + + private List<ToolInfo> tools; } diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/RepoInfo.java b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/RepoInfo.java index 874d886f..e92415b2 100644 --- a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/RepoInfo.java +++ b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/RepoInfo.java @@ -32,6 +32,4 @@ public class RepoInfo { private String arch; private String baseUrl; - - private Integer type; } diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ToolInfo.java similarity index 87% copy from bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java copy to bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ToolInfo.java index 957e0c86..6069809e 100644 --- a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ClusterInfo.java +++ b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/ToolInfo.java @@ -21,11 +21,15 @@ package org.apache.bigtop.manager.grpc.pojo; import lombok.Data; @Data -public class ClusterInfo { +public class ToolInfo { private String name; - private String userGroup; + private String baseUrl; - private String rootDir; + private String pkgName; + + private String arch; + + private String checksum; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java index 9221db33..6abf0f2c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java @@ -25,18 +25,21 @@ import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.po.HostPO; import org.apache.bigtop.manager.dao.po.RepoPO; import org.apache.bigtop.manager.dao.po.ServiceConfigPO; +import org.apache.bigtop.manager.dao.po.ToolPO; import org.apache.bigtop.manager.dao.query.ComponentQuery; import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.dao.repository.ComponentDao; import org.apache.bigtop.manager.dao.repository.HostDao; import org.apache.bigtop.manager.dao.repository.RepoDao; import org.apache.bigtop.manager.dao.repository.ServiceConfigDao; +import org.apache.bigtop.manager.dao.repository.ToolDao; import org.apache.bigtop.manager.grpc.generated.JobCacheReply; import org.apache.bigtop.manager.grpc.generated.JobCacheRequest; import org.apache.bigtop.manager.grpc.generated.JobCacheServiceGrpc; import org.apache.bigtop.manager.grpc.payload.JobCachePayload; import org.apache.bigtop.manager.grpc.pojo.ClusterInfo; import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.ToolInfo; import org.apache.bigtop.manager.server.exception.ServerException; import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.holder.SpringContextHolder; @@ -63,6 +66,7 @@ public class JobCacheHelper { private static RepoDao repoDao; private static HostDao hostDao; private static ComponentDao componentDao; + private static ToolDao toolDao; private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false); @@ -72,6 +76,7 @@ public class JobCacheHelper { repoDao = SpringContextHolder.getBean(RepoDao.class); hostDao = SpringContextHolder.getBean(HostDao.class); componentDao = SpringContextHolder.getBean(ComponentDao.class); + toolDao = SpringContextHolder.getBean(ToolDao.class); INITIALIZED.set(true); } @@ -127,12 +132,25 @@ public class JobCacheHelper { } ClusterPO clusterPO = clusterDao.findById(clusterId); + List<ToolPO> toolPOList = toolDao.findAll(); ClusterInfo clusterInfo = new ClusterInfo(); clusterInfo.setName(clusterPO.getName()); clusterInfo.setUserGroup(clusterPO.getUserGroup()); clusterInfo.setRootDir(clusterPO.getRootDir()); + List<ToolInfo> toolInfoList = new ArrayList<>(); + for (ToolPO toolPO : toolPOList) { + ToolInfo toolInfo = new ToolInfo(); + toolInfo.setName(toolPO.getName()); + toolInfo.setBaseUrl(toolPO.getBaseUrl()); + toolInfo.setPkgName(toolPO.getPkgName()); + toolInfo.setArch(toolPO.getArch()); + toolInfo.setChecksum(toolPO.getChecksum()); + toolInfoList.add(toolInfo); + } + clusterInfo.setTools(toolInfoList); + Map<String, Map<String, String>> serviceConfigMap = payload.getConfigurations(); serviceConfigMap.putAll(getServiceConfigMap(clusterId)); @@ -162,7 +180,6 @@ public class JobCacheHelper { repoInfo.setName(repoPO.getName()); repoInfo.setArch(repoPO.getArch()); repoInfo.setBaseUrl(repoPO.getBaseUrl()); - repoInfo.setType(repoPO.getType()); repoList.add(repoInfo); }); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java index adc9ca28..28c1dd84 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java @@ -34,6 +34,4 @@ public class RepoDTO { private String arch; private String baseUrl; - - private Integer type; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java index 1808cc3f..f1d9a621 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java @@ -218,9 +218,7 @@ public class HostServiceImpl implements HostService { @Override public Boolean installDependencies(List<HostDTO> hostDTOList) { List<RepoPO> repoPOList = repoDao.findAll(); - Map<String, RepoPO> archRepoMap = repoPOList.stream() - .filter(repoPO -> repoPO.getType() == 2) - .collect(Collectors.toMap(RepoPO::getArch, repo -> repo)); + Map<String, RepoPO> archRepoMap = repoPOList.stream().collect(Collectors.toMap(RepoPO::getArch, repo -> repo)); // Clear cache list installedStatus.clear(); diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index 3314dfda..e6add511 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -126,8 +126,7 @@ CREATE TABLE `repo` `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `name` VARCHAR(32) DEFAULT NULL, `arch` VARCHAR(32) DEFAULT NULL, - `base_url` VARCHAR(256) DEFAULT NULL, - `type` INT DEFAULT NULL COMMENT '1-services, 2-tools', + `base_url` VARCHAR(255) DEFAULT NULL, `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `create_by` BIGINT, @@ -332,16 +331,29 @@ CREATE TABLE `llm_chat_message` KEY `idx_user_id` (`user_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE `tool` +( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + `name` VARCHAR(64) DEFAULT NULL, + `base_url` VARCHAR(255) DEFAULT NULL, + `pkg_name` VARCHAR(64) DEFAULT NULL, + `arch` VARCHAR(64) DEFAULT NULL, + `checksum` VARCHAR(255) DEFAULT NULL, + `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, + `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `create_by` BIGINT, + `update_by` BIGINT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- Adding default admin user INSERT INTO user (username, password, nickname, status) VALUES ('admin', '21232f297a57a5a743894a0e4a801fc3', 'Administrator', true); -INSERT INTO repo (name, arch, base_url, type) +INSERT INTO repo (name, arch, base_url) VALUES -('Service tarballs', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tarballs', 1), -('Service tarballs', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tarballs', 1), -('BM tools', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tools', 2), -('BM tools', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tools', 2); +('Service tarballs', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0'), +('Service tarballs', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0'); -- Adding default llm platform INSERT INTO llm_platform (credential, name, support_models) @@ -365,4 +377,10 @@ WHERE `name` = 'QianFan'; UPDATE `llm_platform` SET `desc` = 'Get your API Key in https://platform.deepseek.com' -WHERE `name` = 'DeepSeek'; \ No newline at end of file +WHERE `name` = 'DeepSeek'; + +INSERT INTO tool (name, base_url, pkg_name, arch, checksum) +VALUES +('jdk8', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_x64_linux_hotspot_8u452b09.tar.gz', 'x86_64', 'SHA-256:9448308a21841960a591b47927cf2d44fdc4c0533a5f8111a4b243a6bafb5d27'), +('jdk8', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_aarch64_linux_hotspot_8u452b09.tar.gz', 'aarch64', 'SHA-256:d8a1aecea0913b7a1e0d737ba6f7ea99059b3f6fd17813d4a24e8b3fc3aee278'), +('mysql-connector-j', 'https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/', 'mysql-connector-j-8.0.33.jar', 'x86_64,aarch64', 'SHA-256:e2a3b2fc726a1ac64e998585db86b30fa8bf3f706195b78bb77c5f99bf877bd9'); diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 02f59f44..94f94874 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -120,8 +120,7 @@ CREATE TABLE repo id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, name VARCHAR(32) DEFAULT NULL, arch VARCHAR(32) DEFAULT NULL, - base_url VARCHAR(256) DEFAULT NULL, - type INTEGER DEFAULT NULL, + base_url VARCHAR(255) DEFAULT NULL, create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, create_by BIGINT, @@ -129,8 +128,6 @@ CREATE TABLE repo PRIMARY KEY (id) ); -COMMENT ON COLUMN repo.type IS '1-services, 2-tools'; - CREATE TABLE service ( id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, @@ -343,18 +340,31 @@ CREATE TABLE llm_chat_message PRIMARY KEY (id) ); +CREATE TABLE tool +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(64) NOT NULL, + base_url VARCHAR(255) DEFAULT NULL, + pkg_name VARCHAR(64) DEFAULT NULL, + arch VARCHAR(64) DEFAULT NULL, + checksum VARCHAR(255) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id) +); + CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); CREATE INDEX idx_message_user_id ON llm_chat_message (user_id); INSERT INTO "user" (username, password, nickname, status) VALUES ('admin', '21232f297a57a5a743894a0e4a801fc3', 'Administrator', true); -INSERT INTO repo (name, arch, base_url, type) +INSERT INTO repo (name, arch, base_url) VALUES -('Service tarballs', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tarballs', 1), -('Service tarballs', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tarballs', 1), -('BM tools', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tools', 2), -('BM tools', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0/tools', 2); +('Service tarballs', 'x86_64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0'), +('Service tarballs', 'aarch64', 'http://repos.bigtop.apache.org/releases/bigtop-manager/1.0.0'); INSERT INTO llm_platform (credential, name, support_models) VALUES @@ -377,4 +387,10 @@ WHERE "name" = 'QianFan'; UPDATE llm_platform SET "desc" = 'Get your API Key in https://platform.deepseek.com' -WHERE "name" = 'DeepSeek'; \ No newline at end of file +WHERE "name" = 'DeepSeek'; + +INSERT INTO tool (name, base_url, pkg_name, arch, checksum) +VALUES +('jdk8', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_x64_linux_hotspot_8u452b09.tar.gz', 'x86_64', 'SHA-256:9448308a21841960a591b47927cf2d44fdc4c0533a5f8111a4b243a6bafb5d27'), +('jdk8', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_aarch64_linux_hotspot_8u452b09.tar.gz', 'aarch64', 'SHA-256:d8a1aecea0913b7a1e0d737ba6f7ea99059b3f6fd17813d4a24e8b3fc3aee278'), +('mysql-connector-j', 'https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/', 'mysql-connector-j-8.0.33.jar', 'x86_64,aarch64', 'SHA-256:e2a3b2fc726a1ac64e998585db86b30fa8bf3f706195b78bb77c5f99bf877bd9'); diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/order.json index 459414e4..55f4cf14 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/order.json @@ -1,3 +1,3 @@ { - "FLINK_HISTORY_SERVER-START": ["NAMENODE-START", "DATANODE-START"] + "FLINK_HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"] } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java index 373e5dc5..065a6d2a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java @@ -21,11 +21,12 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hive; import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.constants.MessageConstants; import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.grpc.pojo.ToolInfo; import org.apache.bigtop.manager.stack.core.exception.StackException; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; +import org.apache.bigtop.manager.stack.core.tarball.FileDownloader; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -33,6 +34,7 @@ import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; +import java.io.File; import java.io.IOException; import java.text.MessageFormat; import java.util.Properties; @@ -113,9 +115,9 @@ public class HiveMetastoreScript extends AbstractServerScript { } private void downloadMySQLJdbcDriver(Params params) { - String mysqlDriver = - "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar";; - TarballDownloader.download(mysqlDriver, params.stackHome()); + ToolInfo tool = LocalSettings.getTool("mysql-connector-j"); + String downloadUrl = tool.getBaseUrl() + File.separator + tool.getPkgName(); + FileDownloader.download(downloadUrl, params.stackHome(), tool); LinuxFileUtils.moveFile(params.stackHome() + "/mysql-connector-j-8.0.33.jar", params.serviceHome() + "/lib/"); LinuxFileUtils.updateOwner(params.serviceHome() + "/lib", params.user(), params.group(), true); LinuxFileUtils.updatePermissions(params.serviceHome() + "/lib", Constants.PERMISSION_755, true); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index 640370ab..43c04f2b 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -24,7 +24,7 @@ import org.apache.bigtop.manager.grpc.pojo.RepoInfo; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; +import org.apache.bigtop.manager.stack.core.tarball.FileDownloader; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -52,7 +52,7 @@ public class TezClientScript extends AbstractClientScript { String stackHome = params.stackHome(); for (PackageInfo packageInfo : packages) { String remoteUrl = repoUrl + File.separator + packageInfo.getName(); - TarballDownloader.download(remoteUrl, stackHome, packageInfo); + FileDownloader.download(remoteUrl, stackHome, packageInfo); } return ShellResult.success(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java index 3c7985dc..9f6f47a0 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java @@ -95,8 +95,7 @@ public abstract class BaseParams implements Params { @Override public RepoInfo repo() { return LocalSettings.repos().stream() - // Use service tarballs repo(type == 1) - .filter(r -> OSDetection.getArch().equals(r.getArch()) && r.getType() == 1) + .filter(r -> OSDetection.getArch().equals(r.getArch())) .findFirst() .orElseThrow(() -> new StackException( "Cannot find repo for os: [{0}] and arch: [{1}]", OSDetection.getOS(), OSDetection.getArch())); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/FileDownloader.java similarity index 91% rename from bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java rename to bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/FileDownloader.java index 4e1cf5b0..1e5608ba 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/FileDownloader.java @@ -19,6 +19,7 @@ package org.apache.bigtop.manager.stack.core.tarball; import org.apache.bigtop.manager.grpc.pojo.PackageInfo; +import org.apache.bigtop.manager.grpc.pojo.ToolInfo; import org.apache.bigtop.manager.stack.core.exception.StackException; import lombok.extern.slf4j.Slf4j; @@ -30,7 +31,16 @@ import java.net.HttpURLConnection; import java.net.URL; @Slf4j -public class TarballDownloader { +public class FileDownloader { + + public static void download(String remoteUrl, String saveDir, ToolInfo toolInfo) { + PackageInfo packageInfo = new PackageInfo(); + packageInfo.setUrl(toolInfo.getBaseUrl()); + packageInfo.setName(toolInfo.getPkgName()); + packageInfo.setChecksum(toolInfo.getChecksum()); + + download(remoteUrl, saveDir, packageInfo); + } @SuppressWarnings("ResultOfMethodCallIgnored") public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java index ba0b7363..7e90001d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java @@ -21,8 +21,11 @@ package org.apache.bigtop.manager.stack.core.utils; import org.apache.bigtop.manager.common.constants.CacheFiles; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.common.utils.ProjectPathUtils; +import org.apache.bigtop.manager.common.utils.os.OSDetection; import org.apache.bigtop.manager.grpc.pojo.ClusterInfo; import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.ToolInfo; +import org.apache.bigtop.manager.stack.core.exception.StackException; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; @@ -41,7 +44,6 @@ public class LocalSettings { } public static Map<String, Object> configurations(String service, String type) { - Map<String, Object> configDataMap = new HashMap<>(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.CONFIGURATIONS_INFO); try { @@ -65,7 +67,6 @@ public class LocalSettings { } public static Map<String, List<String>> hosts() { - Map<String, List<String>> hostJson = new HashMap<>(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.HOSTS_INFO); if (file.exists()) { @@ -75,7 +76,6 @@ public class LocalSettings { } public static Map<String, Object> basicInfo() { - Map<String, Object> settings = new HashMap<>(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.SETTINGS_INFO); if (file.exists()) { @@ -85,7 +85,6 @@ public class LocalSettings { } public static Map<String, String> users() { - Map<String, String> userMap = new HashMap<>(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.USERS_INFO); if (file.exists()) { @@ -99,7 +98,6 @@ public class LocalSettings { } public static List<RepoInfo> repos() { - List<RepoInfo> repoInfoList = List.of(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.REPOS_INFO); if (file.exists()) { @@ -109,7 +107,6 @@ public class LocalSettings { } public static ClusterInfo cluster() { - ClusterInfo clusterInfo = new ClusterInfo(); File file = createFile(ProjectPathUtils.getAgentCachePath() + CacheFiles.CLUSTER_INFO); if (file.exists()) { @@ -118,6 +115,22 @@ public class LocalSettings { return clusterInfo; } + public static ToolInfo getTool(String name) { + return getTool(name, OSDetection.getArch()); + } + + public static ToolInfo getTool(String name, String arch) { + ClusterInfo clusterInfo = cluster(); + for (ToolInfo toolInfo : clusterInfo.getTools()) { + if (toolInfo.getName().equals(name) && toolInfo.getArch().contains(arch)) { + return toolInfo; + } + } + + log.error("Cannot find tool: [{}] for arch: [{}]", name, arch); + throw new StackException("Tool not found: " + name); + } + protected static File createFile(String fileName) { return new File(fileName); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java index 263694e2..bed366e6 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.core.utils; import org.apache.bigtop.manager.grpc.pojo.PackageInfo; -import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; +import org.apache.bigtop.manager.stack.core.tarball.FileDownloader; import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor; import org.apache.commons.lang3.StringUtils; @@ -44,7 +44,7 @@ public class TarballUtils { ? repoUrl + File.separator + packageInfo.getName() : packageInfo.getUrl() + File.separator + packageInfo.getName(); File localFile = new File(stackHome + File.separator + packageInfo.getName()); - TarballDownloader.download(remoteUrl, stackHome, packageInfo); + FileDownloader.download(remoteUrl, stackHome, packageInfo); log.info("Extracting [{}] to [{}]", localFile.getAbsolutePath(), serviceHome); TarballExtractor.extractTarball(localFile.getAbsolutePath(), serviceHome, skipLevels); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java index f3dfb8ed..59d53a81 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParamsTest.java @@ -57,9 +57,8 @@ public class BaseParamsTest { @BeforeEach public void setUp() { List<RepoInfo> repos = new ArrayList<>(); - repos.add(new RepoInfo("repo1", "mockArch", "testURL", 2)); - repos.add(new RepoInfo("repo2", "mockArch", "testURL", 1)); - repos.add(new RepoInfo("repo3", "mockArch", "testURL", 1)); + repos.add(new RepoInfo("repo2", "mockArch", "testURL")); + repos.add(new RepoInfo("repo3", "mockArch", "testURL")); List<String> arch = new ArrayList<>(); arch.add("mockArch"); @@ -133,7 +132,6 @@ public class BaseParamsTest { @Test public void testRepo() { RepoInfo repo = mockBaseParams.repo(); - assertEquals(1, repo.getType()); assertEquals("repo2", repo.getName()); assertEquals("mockArch", repo.getArch()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/utils/LocalSettingsTest.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/utils/LocalSettingsTest.java index fef270bf..b5f31d45 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/utils/LocalSettingsTest.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/test/java/org/apache/bigtop/manager/stack/core/utils/LocalSettingsTest.java @@ -174,8 +174,8 @@ public class LocalSettingsTest { @Test public void testRepos() { - RepoInfo repo1 = new RepoInfo("repo1", "x86_64", "http://repo1.com";, 1); - RepoInfo repo2 = new RepoInfo("repo2", "arch64", "http://repo2.com";, 2); + RepoInfo repo1 = new RepoInfo("repo1", "x86_64", "http://repo1.com";); + RepoInfo repo2 = new RepoInfo("repo2", "arch64", "http://repo2.com";); List<RepoInfo> repoInfoList = List.of(repo1, repo2); File file = mock(File.class); @@ -190,8 +190,8 @@ public class LocalSettingsTest { localSettingsMockedStatic.when(LocalSettings::repos).thenCallRealMethod(); List<RepoInfo> expectedRepoInfoList = List.of( - new RepoInfo("repo1", "x86_64", "http://repo1.com";, 1), - new RepoInfo("repo2", "arch64", "http://repo2.com";, 2)); + new RepoInfo("repo1", "x86_64", "http://repo1.com";), + new RepoInfo("repo2", "arch64", "http://repo2.com";)); assertEquals(expectedRepoInfoList, LocalSettings.repos()); } diff --git a/docs/en/cluster.md b/docs/en/cluster.md index a16e0950..da0b6440 100644 --- a/docs/en/cluster.md +++ b/docs/en/cluster.md @@ -19,12 +19,12 @@ The stack page is a display page that mainly shows which optional services are a  -Expect for our official repository, users can also set their own repository which can configured here +Expect for our official repository, users can also set their own repository which can configured here(due to license issue, MySQL/Grafana in Infra Stack are downloaded from the official website instead of the repository). We also provide other ways for you download dependencies: * BaiduNetdisk: https://pan.baidu.com/s/162FXYsaRuwFQjrOlMuDRjg?pwd=hufb - + ### Hosts #### Add a Host diff --git a/docs/zh/cluster.md b/docs/zh/cluster.md index d54b550b..3dab536c 100644 --- a/docs/zh/cluster.md +++ b/docs/zh/cluster.md @@ -17,12 +17,12 @@  -除了官方 Repository 外，用户也可以搭建自己的 Repository 并在此处进行配置 +除了官方 Repository 外，用户也可以搭建自己的 Repository 并在此处进行配置（注意 Infra 中的 MySQL/Grafana 由于 License 原因，是从官方网站下载而不是从 Repository 下载） 我们也提供了其他方式供您下载相应的依赖： * 百度云：https://pan.baidu.com/s/162FXYsaRuwFQjrOlMuDRjg?pwd=hufb - + ### 主机 #### 新增主机
