This is an automated email from the ASF dual-hosted git repository. pengjunzhi pushed a commit to branch avoid-800k in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit cc0938220ade7b1d13f12a4392746399578d496c Author: Peng Junzhi <[email protected]> AuthorDate: Sat Nov 8 18:30:05 2025 +0800 introducing auto-download .so and makefile --- vermeer/.gitignore | 11 ++ vermeer/Makefile | 85 ++++++++++++++++ vermeer/README.md | 74 ++++++++++---- vermeer/README.zh-CN.md | 72 +++++++++---- vermeer/build.sh | 10 ++ vermeer/scripts/download_binaries.sh | 190 +++++++++++++++++++++++++++++++++++ 6 files changed, 398 insertions(+), 44 deletions(-) diff --git a/vermeer/.gitignore b/vermeer/.gitignore index 53629789..0c7edade 100644 --- a/vermeer/.gitignore +++ b/vermeer/.gitignore @@ -87,3 +87,14 @@ node_modules/ # Others # ###################### test/case/ + +# Downloaded binaries (should be downloaded via scripts/download_binaries.sh) # +###################### +tools/supervisord/*/supervisord +tools/protoc/*/protoc +tools/protoc/*/bin/ +tools/protoc/*/include/ + +# Generated files (should be generated via go generate) # +###################### +asset/assets_vfsdata.go diff --git a/vermeer/Makefile b/vermeer/Makefile new file mode 100644 index 00000000..422d3152 --- /dev/null +++ b/vermeer/Makefile @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: all init download-binaries generate-assets build clean help + +# Default target +all: generate-assets build + +# Initialize project (first time setup) +init: download-binaries + @echo "Installing Go dependencies..." + go mod download + @echo "Project initialized successfully!" + +# Download binary dependencies (supervisord, protoc) +download-binaries: + @echo "Downloading binary dependencies..." + @./scripts/download_binaries.sh + +# Generate assets (vfsdata.go for web UI) +generate-assets: + @echo "Generating assets..." + @cd asset && go generate + @echo "Assets generated successfully!" + +# Build vermeer binary +build: + @echo "Building vermeer..." + @go build -o vermeer + @echo "Build completed: ./vermeer" + +# Build for specific platform +build-linux-amd64: + @echo "Building for linux/amd64..." + @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o vermeer + +build-linux-arm64: + @echo "Building for linux/arm64..." + @CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o vermeer + +# Clean generated files and binaries +clean: + @echo "Cleaning..." + @rm -f vermeer + @rm -f asset/assets_vfsdata.go + @echo "Clean completed!" + +# Clean including downloaded binaries +clean-all: clean + @echo "Cleaning downloaded binaries..." + @rm -rf tools/supervisord/*/supervisord + @rm -rf tools/protoc/*/protoc + @rm -rf tools/protoc/*/bin + @rm -rf tools/protoc/*/include + @echo "All clean completed!" + +# Help +help: + @echo "Vermeer Build System" + @echo "" + @echo "Usage:" + @echo " make init - First time setup (download binaries + go mod download)" + @echo " make download-binaries - Download supervisord and protoc binaries" + @echo " make generate-assets - Generate assets_vfsdata.go from web UI" + @echo " make build - Build vermeer for current platform" + @echo " make build-linux-amd64 - Build for Linux AMD64" + @echo " make build-linux-arm64 - Build for Linux ARM64" + @echo " make clean - Remove generated files and binaries" + @echo " make clean-all - Remove everything including downloaded tools" + @echo " make all - Generate assets and build (default)" + @echo " make help - Show this help message" diff --git a/vermeer/README.md b/vermeer/README.md index 55ca14b0..5a74bd90 100644 --- a/vermeer/README.md +++ b/vermeer/README.md @@ -48,42 +48,72 @@ Configuration file reference config/supervisor.conf ./supervisord -c supervisor.conf -d ```` -## Compile -Required -* go 1.23 +## Build from Source -### Install dependencies +### Requirements +* Go 1.23 or later +* `curl` and `unzip` utilities (for downloading dependencies) +* Internet connection (for first-time setup) +### Quick Start + +**Recommended**: Use Makefile for building: + +```bash +# First time setup (downloads binary dependencies) +make init + +# Build vermeer +make ``` -go mod tidy + +**Alternative**: Use the build script: + +```bash +# For AMD64 +./build.sh amd64 + +# For ARM64 +./build.sh arm64 ``` -### Local compile +The build process will automatically: +1. Download required binary tools (supervisord, protoc) +2. Generate web UI assets +3. Build the vermeer binary + +### Build Targets +```bash +make build # Build for current platform +make build-linux-amd64 # Build for Linux AMD64 +make build-linux-arm64 # Build for Linux ARM64 +make clean # Clean generated files +make help # Show all available targets ``` -go build + +### Development Build + +For development with hot-reload of web UI: + +```bash +go build -tags=dev ``` --- -### install grpc protobuf dependencies -```` -go install google.golang.org/protobuf/cmd/[email protected] \ -go install google.golang.org/grpc/cmd/[email protected] -```` - -### protobuf build -```` -../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=. -```` +### Protobuf Development +If you need to regenerate protobuf files: -### Cross Compile +```bash +# Install protobuf Go plugins +go install google.golang.org/protobuf/cmd/[email protected] +go install google.golang.org/grpc/cmd/[email protected] -```` -linux: GOARCH=amd64 GOOS=linux go build -CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin -```` +# Generate protobuf files +tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=. +``` --- diff --git a/vermeer/README.zh-CN.md b/vermeer/README.zh-CN.md index 1b125fa3..45ed2c00 100644 --- a/vermeer/README.zh-CN.md +++ b/vermeer/README.zh-CN.md @@ -48,44 +48,72 @@ worker: ./vermeer --env=worker01 ./supervisord -c supervisor.conf -d ```` -## 编译 +## 从源码编译 -* Go 1.23 +### 环境要求 +* Go 1.23 或更高版本 +* `curl` 和 `unzip` 工具(用于下载依赖) +* 互联网连接(首次构建时需要) -### 安装依赖项 +### 快速开始 -``` -go mod tidy +**推荐**: 使用 Makefile 进行构建: + +```bash +# 首次设置(下载二进制依赖) +make init + +# 构建 vermeer +make ``` -### 本地编译 +**替代方案**: 使用构建脚本: +```bash +# AMD64 架构 +./build.sh amd64 + +# ARM64 架构 +./build.sh arm64 ``` -go build + +构建过程会自动: +1. 下载所需的二进制工具(supervisord, protoc) +2. 生成 Web UI 资源文件 +3. 构建 vermeer 二进制文件 + +### 构建目标 + +```bash +make build # 为当前平台构建 +make build-linux-amd64 # 为 Linux AMD64 构建 +make build-linux-arm64 # 为 Linux ARM64 构建 +make clean # 清理生成的文件 +make help # 显示所有可用目标 ``` -### grpc protobuf 依赖项安装 -```` -go install google.golang.org/protobuf/cmd/[email protected] \ -go install google.golang.org/grpc/cmd/[email protected] -```` +### 开发构建 +如需在开发环境中热重载 Web UI: +```bash +go build -tags=dev +``` -### protobuf build -生成protobuf文件 -```` -../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=. -```` +--- +### Protobuf 开发 +如需重新生成 protobuf 文件: -### 交叉编译 +```bash +# 安装 protobuf Go 插件 +go install google.golang.org/protobuf/cmd/[email protected] +go install google.golang.org/grpc/cmd/[email protected] -```` -linux: GOARCH=amd64 GOOS=linux go build -CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin -```` +# 生成 protobuf 文件 +tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=. +``` --- diff --git a/vermeer/build.sh b/vermeer/build.sh index c2f48070..47cd42e3 100644 --- a/vermeer/build.sh +++ b/vermeer/build.sh @@ -28,6 +28,16 @@ go env -w GONOSUMDB=\* ## 目前有一些代码库还 #go env -w CXX=/opt/compiler/gcc-8.2/bin/g++ go mod download + +# Download binary dependencies if not exist +echo "Checking binary dependencies..." +./scripts/download_binaries.sh + +# Generate assets if not exist +if [ ! -f "asset/assets_vfsdata.go" ]; then + echo "Generating assets..." + cd asset && go generate && cd .. +fi ARCH=$1 CGO_ENABLED=0 GOOS=linux GOARCH="$ARCH" go build diff --git a/vermeer/scripts/download_binaries.sh b/vermeer/scripts/download_binaries.sh new file mode 100755 index 00000000..13ffb0eb --- /dev/null +++ b/vermeer/scripts/download_binaries.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +TOOLS_DIR="$PROJECT_ROOT/tools" + +# Versions +SUPERVISORD_VERSION="4.2.5" +PROTOC_VERSION="21.12" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Download and verify file with MD5 checksum +download_and_verify() { + local url=$1 + local filepath=$2 + local expected_md5=$3 + local md5_cmd + + # Detect md5 command (md5sum on Linux, md5 on macOS) + if command -v md5sum &> /dev/null; then + md5_cmd="md5sum" + elif command -v md5 &> /dev/null; then + md5_cmd="md5 -r" + else + log_warn "MD5 verification tool not found, skipping checksum verification" + expected_md5="" + fi + + if [[ -f $filepath ]]; then + if [[ -n $expected_md5 ]]; then + log_info "File $filepath exists. Verifying MD5 checksum..." + actual_md5=$($md5_cmd "$filepath" | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + log_warn "MD5 checksum mismatch for $filepath. Expected: $expected_md5, got: $actual_md5" + log_info "Deleting and re-downloading $filepath..." + rm -f "$filepath" + else + log_info "MD5 checksum verified for $filepath" + return 0 + fi + else + log_info "File $filepath already exists, skipping download" + return 0 + fi + fi + + log_info "Downloading $filepath..." + if ! curl -L -f "$url" -o "$filepath"; then + log_error "Failed to download from $url" + return 1 + fi + + if [[ -n $expected_md5 ]]; then + actual_md5=$($md5_cmd "$filepath" | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + log_error "MD5 checksum verification failed after download. Expected: $expected_md5, got: $actual_md5" + rm -f "$filepath" + return 1 + fi + log_info "MD5 checksum verified successfully" + fi + + return 0 +} + +# Download supervisord +download_supervisord() { + local platform=$1 + local arch=$2 + local md5=$3 + + SUPERVISORD_DIR="$TOOLS_DIR/supervisord/${platform}" + mkdir -p "$SUPERVISORD_DIR" + + local download_url="https://github.com/ochinchina/supervisord/releases/download/v${SUPERVISORD_VERSION}/supervisord_${SUPERVISORD_VERSION}_Linux_${arch}.tar.gz"; + local temp_file="/tmp/supervisord_${platform}.tar.gz" + + log_info "Downloading supervisord for ${platform}..." + + if ! download_and_verify "$download_url" "$temp_file" "$md5"; then + return 1 + fi + + if [ ! -f "$SUPERVISORD_DIR/supervisord" ]; then + tar -xzf "$temp_file" -C "$SUPERVISORD_DIR" --strip-components=1 + chmod +x "$SUPERVISORD_DIR/supervisord" + log_info "Successfully extracted supervisord for ${platform}" + fi + + rm -f "$temp_file" + return 0 +} + +# Download protoc +download_protoc() { + local platform=$1 + local protoc_platform=$2 + local md5=$3 + + PROTOC_DIR="$TOOLS_DIR/protoc/${platform}" + mkdir -p "$PROTOC_DIR" + + local download_url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${protoc_platform}.zip"; + local temp_file="/tmp/protoc_${platform}.zip" + + log_info "Downloading protoc for ${platform}..." + + if ! download_and_verify "$download_url" "$temp_file" "$md5"; then + return 1 + fi + + if [ ! -f "$PROTOC_DIR/protoc" ]; then + unzip -q "$temp_file" -d "$PROTOC_DIR" + chmod +x "$PROTOC_DIR/bin/protoc" + + # Move protoc binary to the root of protoc directory for compatibility + if [ -f "$PROTOC_DIR/bin/protoc" ]; then + cp "$PROTOC_DIR/bin/protoc" "$PROTOC_DIR/protoc" + fi + + log_info "Successfully extracted protoc for ${platform}" + fi + + rm -f "$temp_file" + return 0 +} + +# Main function +main() { + log_info "Starting to download binary dependencies..." + log_info "Tools directory: $TOOLS_DIR" + + # Download supervisord for different platforms + # MD5 checksums for supervisord v4.2.5 + download_supervisord "linux_amd64" "x86_64" "" # Add MD5 if available + download_supervisord "linux_arm64" "aarch64" "" # Add MD5 if available + + # Download protoc for different platforms + # MD5 checksums for protoc v21.12 + download_protoc "linux64" "linux-x86_64" "" # Add MD5 if available + download_protoc "osxm1" "osx-aarch_64" "" # Add MD5 if available + download_protoc "win64" "win64" "" # Add MD5 if available + + log_info "All binary dependencies downloaded successfully!" + log_info "" + log_info "Downloaded binaries:" + log_info " - supervisord (linux_amd64, linux_arm64)" + log_info " - protoc (linux64, osxm1, win64)" + log_info "" + log_info "Note: These binaries are excluded from source releases." + log_info "Users should run 'make download-binaries' before building." +} + +# Run main function +main "$@"
