This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch building-system
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit cb1a4df9ee66b457ba99191576658b264128b48f
Author: Gao Hongtao <hanahm...@gmail.com>
AuthorDate: Tue Aug 27 19:02:30 2024 +0800
Refactor building system
Signed-off-by: Gao Hongtao <hanahm...@gmail.com>
---
.github/workflows/publish-docker.yml | 8 +-
Makefile | 16 +---
{docker => banyand}/Dockerfile | 42 ++-------
banyand/Makefile | 4 +-
banyand/k8s.yml | 169 ----------------------------------
banyand/measure/datapoints_test.go | 71 ++++++++++++++
banyand/okteto.yml | 34 -------
docker/Makefile => bydbctl/Dockerfile | 22 ++++-
bydbctl/Makefile | 37 ++------
bydbctl/cmd/{bydbctl => cli}/main.go | 0
docker/.dockerignore | 31 -------
docs/installation/binaries.md | 63 +++++++++----
scripts/build/base.mk | 43 +++++++++
scripts/build/build.mk | 105 ++++++++++++---------
scripts/build/docker.mk | 7 +-
scripts/release.sh | 34 +++++--
16 files changed, 297 insertions(+), 389 deletions(-)
diff --git a/.github/workflows/publish-docker.yml
b/.github/workflows/publish-docker.yml
index f5120fcd..77bd45cb 100644
--- a/.github/workflows/publish-docker.yml
+++ b/.github/workflows/publish-docker.yml
@@ -75,12 +75,16 @@ jobs:
run: make generate
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
+ - name: Build Linux binaries
+ run: |
+ TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release
- name: Build Windows binaries
run: |
- GOOS=windows GOARCH=amd64 make -C banyand banyand-server-static
- GOOS=windows GOARCH=amd64 make -C bydbctl build
+ TARGET_OS=windows PLATFORMS=windows/amd64 make release
- name: Build docker image
if: github.ref != 'refs/heads/main' # Only build docker image on
PR(Push image when pushed to main branch)
+ env:
+ CI: false
run: |
make docker.build || make docker.build
docker image ls
diff --git a/Makefile b/Makefile
index 6e94331a..3bc3dbaf 100644
--- a/Makefile
+++ b/Makefile
@@ -149,20 +149,12 @@ license-dep: default ## Fix license header issues
##@ Docker targets
docker.build: TARGET=docker
-docker.build: DIR=docker
-docker.build:
- $(MAKE) $(TARGET) -C $(DIR); \
- if [ $$? -ne 0 ]; then \
- exit 1; \
- fi; \
+docker.build: PROJECTS:= banyand bydbctl
+docker.build: default ## Build docker images
docker.push: TARGET=docker.push
-docker.push: DIR=docker
-docker.push:
- $(MAKE) $(TARGET) -C $(DIR); \
- if [ $$? -ne 0 ]; then \
- exit 1; \
- fi; \
+docker.push: PROJECTS:= banyand bydbctl
+docker.push: default ## Push docker images
default:
@for PRJ in $(PROJECTS); do \
diff --git a/docker/Dockerfile b/banyand/Dockerfile
similarity index 56%
rename from docker/Dockerfile
rename to banyand/Dockerfile
index de9118ab..0ed89554 100644
--- a/docker/Dockerfile
+++ b/banyand/Dockerfile
@@ -14,54 +14,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM golang:1.22 AS dev
-WORKDIR /app
-ENV GOOS="linux"
-ENV CGO_ENABLED=0
-
-RUN go install github.com/cosmtrek/air@latest \
- && go install github.com/go-delve/delve/cmd/dlv@latest
-
-EXPOSE 8080
-EXPOSE 2345
-
-ENTRYPOINT ["air"]
-
-FROM golang:1.22 AS base
-
-ENV GOPATH "/go"
-ENV GO111MODULE "on"
-WORKDIR /src
-COPY go.* ./
-RUN go mod download
-
-FROM base AS builder
-
-RUN --mount=target=. \
- --mount=type=cache,target=/root/.cache/go-build \
- BUILD_DIR=/out make -C banyand banyand-server-static
-RUN --mount=target=. \
- --mount=type=cache,target=/root/.cache/go-build \
- BUILD_DIR=/out make -C bydbctl build
FROM alpine:edge AS certs
RUN apk add --no-cache ca-certificates && update-ca-certificates
FROM busybox:stable-glibc as build-linux
-COPY --from=builder /out/banyand-server-static /banyand
+ARG TARGETARCH
+
+COPY build/bin/linux/${TARGETARCH}/banyand-server-static /banyand
COPY --from=certs /etc/ssl/certs /etc/ssl/certs
-COPY --from=builder /out/bydbctl /bydbctl
FROM mcr.microsoft.com/windows/servercore:ltsc2022 as build-windows
-COPY banyand/build/bin/banyand-server-static "/banyand"
-COPY bydbctl/build/bin/bydbctl "/bydbctl"
+ARG TARGETARCH
+
+COPY build/bin/windows/${TARGETARCH}/banyand-server-static "/banyand"
FROM build-${TARGETOS} AS final
EXPOSE 17912
EXPOSE 17913
EXPOSE 6060
+EXPOSE 2121
-ENTRYPOINT ["/banyand"]
\ No newline at end of file
+ENTRYPOINT ["/banyand"]
diff --git a/banyand/Makefile b/banyand/Makefile
index f071b537..48e56a31 100644
--- a/banyand/Makefile
+++ b/banyand/Makefile
@@ -19,7 +19,6 @@
NAME := banyand
SERVER := $(NAME)-server
BINARIES := $(SERVER)
-DEBUG_BINARIES := $(SERVER)-debug
IMG_NAME := skywalking-banyandb
@@ -27,7 +26,10 @@ include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
include ../scripts/build/build.mk
+include ../scripts/build/docker.mk
include ../scripts/build/test.mk
include ../scripts/build/lint.mk
include ../scripts/build/vendor.mk
include ../scripts/build/help.mk
+
+prepare-build: generate
diff --git a/banyand/k8s.yml b/banyand/k8s.yml
deleted file mode 100644
index a416b7a6..00000000
--- a/banyand/k8s.yml
+++ /dev/null
@@ -1,169 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: banyand-metadata
-spec:
- resources:
- requests:
- storage: 5Gi
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
-
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: banyand-measure
-spec:
- resources:
- requests:
- storage: 100Gi
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: banyand-stream
-spec:
- resources:
- requests:
- storage: 100Gi
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: banyand
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: banyand
- strategy:
- type: Recreate
- template:
- metadata:
- labels:
- app: banyand
- spec:
- initContainers:
- - image: busybox
- command: ["/bin/sh"]
- args: ["-c", "rm -rf /tmp/measure/* && rm -rf /tmp/stream/*"]
- name: cleanup
- volumeMounts:
- - name: measure
- mountPath: /tmp/measure
- - name: stream
- mountPath: /tmp/stream
- containers:
- - name: banyand
- image: apache/skywalking-banyandb:v0.0.0-dev
- args:
- - "standalone"
- - "--measure-idx-batch-wait-sec=30"
- - "--logging-level=info"
- imagePullPolicy: Always
- livenessProbe:
- failureThreshold: 5
- httpGet:
- path: /api/healthz
- port: 17913
- scheme: HTTP
- initialDelaySeconds: 20
- periodSeconds: 10
- successThreshold: 1
- timeoutSeconds: 10
- resources:
- limits:
- memory: "8G"
- cpu: "4"
- ports:
- - containerPort: 17912
- - containerPort: 17913
- - containerPort: 2121
- - containerPort: 6060
- volumeMounts:
- - name: metadata
- mountPath: /tmp/metadata
- - name: measure
- mountPath: /tmp/measure
- - name: stream
- mountPath: /tmp/stream
- - image: busybox
- command: ["/bin/sh"]
- args: ["-c", "while true; do ls /tmp; sleep 300s;done"]
- name: debug-entry
- # resources:
- # limits:
- # memory: "10"
- # cpu: "100mi"
- volumeMounts:
- - name: metadata
- mountPath: /tmp/metadata
- - name: measure
- mountPath: /tmp/measure
- - name: stream
- mountPath: /tmp/stream
- nodeSelector:
- group: banyandb
- tolerations:
- - effect: NoSchedule
- key: group
- operator: Equal
- value: banyandb
- volumes:
- - name: metadata
- persistentVolumeClaim:
- claimName: banyand-metadata
- - name: measure
- persistentVolumeClaim:
- claimName: banyand-measure
- - name: stream
- persistentVolumeClaim:
- claimName: banyand-stream
-
----
-apiVersion: v1
-kind: Service
-metadata:
- name: banyand
-spec:
- selector:
- app: banyand
- ports:
- - port: 17912
- targetPort: 17912
-
----
-apiVersion: v1
-kind: Service
-metadata:
- name: api
-spec:
- selector:
- app: banyand
- ports:
- - port: 17913
- targetPort: 17913
- type: LoadBalancer
diff --git a/banyand/measure/datapoints_test.go
b/banyand/measure/datapoints_test.go
new file mode 100644
index 00000000..769698a0
--- /dev/null
+++ b/banyand/measure/datapoints_test.go
@@ -0,0 +1,71 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "sort"
+ "testing"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+)
+
+func TestSorting(t *testing.T) {
+ dp := &dataPoints{
+ seriesIDs: []common.SeriesID{3, 1, 1, 2},
+ timestamps: []int64{300, 100, 100, 200},
+ versions: []int64{3, 1, 3, 2},
+ tagFamilies: [][]nameValues{{}, {}, {}, {}},
+ fields: []nameValues{{}, {}, {}, {}},
+ }
+
+ sort.Sort(dp)
+
+ expectedSeriesIDs := []common.SeriesID{1, 1, 2, 3}
+ expectedTimestamps := []int64{100, 100, 200, 300}
+ expectedVersions := []int64{3, 1, 2, 3}
+
+ for i := range dp.seriesIDs {
+ if dp.seriesIDs[i] != expectedSeriesIDs[i] {
+ t.Errorf("Expected seriesID at index %d to be %d, got
%d", i, expectedSeriesIDs[i], dp.seriesIDs[i])
+ }
+ if dp.timestamps[i] != expectedTimestamps[i] {
+ t.Errorf("Expected timestamp at index %d to be %d, got
%d", i, expectedTimestamps[i], dp.timestamps[i])
+ }
+ if dp.versions[i] != expectedVersions[i] {
+ t.Errorf("Expected version at index %d to be %d, got
%d", i, expectedVersions[i], dp.versions[i])
+ }
+ }
+
+ dp.skip(1)
+
+ expectedSeriesIDs = []common.SeriesID{1, 2, 3}
+ expectedTimestamps = []int64{100, 200, 300}
+ expectedVersions = []int64{3, 2, 3}
+
+ for i := range dp.seriesIDs {
+ if dp.seriesIDs[i] != expectedSeriesIDs[i] {
+ t.Errorf("Expected seriesID at index %d to be %d, got
%d", i, expectedSeriesIDs[i], dp.seriesIDs[i])
+ }
+ if dp.timestamps[i] != expectedTimestamps[i] {
+ t.Errorf("Expected timestamp at index %d to be %d, got
%d", i, expectedTimestamps[i], dp.timestamps[i])
+ }
+ if dp.versions[i] != expectedVersions[i] {
+ t.Errorf("Expected version at index %d to be %d, got
%d", i, expectedVersions[i], dp.versions[i])
+ }
+ }
+}
diff --git a/banyand/okteto.yml b/banyand/okteto.yml
deleted file mode 100644
index 936fdd01..00000000
--- a/banyand/okteto.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
- banyand:
- image: apache/skywalking-banyandb:v0.0.0-dev
- context: ..
-
-deploy:
- - kubectl apply -f k8s.yml
-
-dev:
- banyand:
- image: okteto/golang:1
- command: bash
- sync:
- - ..:/usr/src/app
- volumes:
- - /go
- - /root/.cache
- forward:
- - 2345:2345
\ No newline at end of file
diff --git a/docker/Makefile b/bydbctl/Dockerfile
similarity index 62%
rename from docker/Makefile
rename to bydbctl/Dockerfile
index bb89a358..8f7bc882 100644
--- a/docker/Makefile
+++ b/bydbctl/Dockerfile
@@ -14,5 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-IMG_NAME := skywalking-banyandb
-include ../scripts/build/docker.mk
\ No newline at end of file
+
+FROM alpine:edge AS certs
+RUN apk add --no-cache ca-certificates && update-ca-certificates
+
+FROM busybox:stable-glibc as build-linux
+
+ARG TARGETARCH
+
+COPY build/bin/linux/${TARGETARCH}/bydbctl-cli-static /bydbctl
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2022 as build-windows
+
+ARG TARGETARCH
+
+COPY build/bin/windows/${TARGETARCH}/bydbctl-cli-static "/bydbctl"
+
+FROM build-${TARGETOS} AS final
+
+ENTRYPOINT ["/bydbctl"]
diff --git a/bydbctl/Makefile b/bydbctl/Makefile
index 249207a8..f691b96b 100644
--- a/bydbctl/Makefile
+++ b/bydbctl/Makefile
@@ -17,41 +17,18 @@
#
NAME := bydbctl
-BUILD_DIR ?= build/bin
+BINARIES := $(NAME)-cli
+
+IMG_NAME := skywalking-bydbctl
include ../scripts/build/version.mk
include ../scripts/build/base.mk
include ../scripts/build/generate_go.mk
-
-GOOS ?= linux
-GOARCH ?= amd64
-
-.PHONY: all
-all: build
-
-.PHONY: build
-build:
- @echo "Building binary"
- GOOS=$(GOOS) GOARCH=$(GOARCH) go build -v --ldflags
'${GO_LINK_VERSION}' -o $(BUILD_DIR)/$(NAME)
github.com/apache/skywalking-banyandb/bydbctl/cmd/bydbctl/$*
- chmod +x $(BUILD_DIR)/$(NAME)
- @echo "Done building $(NAME)"
-
-BUILDS := darwin-amd64 darwin-arm64 linux-386 linux-amd64 linux-arm64
windows-386 windows-amd64
-BUILD_RULE = GOOS=$(GOOS) GOARCH=$(GOARCH) \
- go build -v --ldflags '$(GO_LINK_VERSION)' \
- -o $(BUILD_DIR)/$(NAME)-$(VERSION_STRING)-$(GOOS)-$(GOARCH) \
- github.com/apache/skywalking-banyandb/bydbctl/cmd/bydbctl/$*
-
-.PHONY: $(BUILDS)
-$(BUILDS): GOOS = $(word 1,$(subst -, ,$@))
-$(BUILDS): GOARCH = $(word 2,$(subst -, ,$@))
-$(BUILDS):
- $(BUILD_RULE)
-
-.PHONY: release
-release: $(BUILDS)
-
+include ../scripts/build/build.mk
+include ../scripts/build/docker.mk
include ../scripts/build/test.mk
include ../scripts/build/lint.mk
include ../scripts/build/vendor.mk
include ../scripts/build/help.mk
+
+prepare-build: generate
diff --git a/bydbctl/cmd/bydbctl/main.go b/bydbctl/cmd/cli/main.go
similarity index 100%
rename from bydbctl/cmd/bydbctl/main.go
rename to bydbctl/cmd/cli/main.go
diff --git a/docker/.dockerignore b/docker/.dockerignore
deleted file mode 100644
index f52bea00..00000000
--- a/docker/.dockerignore
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to Apache Software Foundation (ASF) under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Apache Software Foundation (ASF) licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-
-bin
-include
-build
-target
-test
-
-dist
-*/node_modules
diff --git a/docs/installation/binaries.md b/docs/installation/binaries.md
index c6659374..258f1c85 100644
--- a/docs/installation/binaries.md
+++ b/docs/installation/binaries.md
@@ -6,7 +6,7 @@ This page shows how to get binaries of Banyand.
Go to the [SkyWalking download
page](https://skywalking.apache.org/downloads/#Database) .
-Select and download the distribution from the suggested location for your
platform, such as `skywalking-banyandb-x.x.x-bin.tgz`.
+Select and download the distribution from the suggested location for your
platform, such as `skywalking-banyandb-x.x.x-bin.tgz` and
`skywalking-bydbctl-x.x.x-bin.tgz`.
> It is essential that you verify the integrity of the downloaded file using
> the PGP signature ( .asc file) or a hash ( .md5 or .sha* file).
@@ -57,44 +57,69 @@ BanyanDB is built on Linux and macOS that introduced
several platform-specific c
To issue the below command to get basic binaries of banyand and bydbctl.
```shell
-$ make generate
+make generate
...
-$ make build
+make build
...
--- banyand: all ---
-make[1]: Entering directory '<path_to_project_root>/banyand'
...
-chmod +x build/bin/banyand-server
-Done building banyand server
-make[1]: Leaving directory '<path_to_project_root>/banyand'
+chmod +x build/bin/banyand-server;
+Done building build/bin/dev/banyand-server
...
--- bydbctl: all ---
-make[1]: Entering directory '<path_to_project_root>/bydbctl'
...
-chmod +x build/bin/bydbctl
-Done building bydbctl
-make[1]: Leaving directory '<path_to_project_root>/bydbctl'
+chmod +x build/bin/dev/bydbctl-cli;
+Done building build/bin/dev/bydbctl-cli
```
The build system provides a series of binary options as well.
* `make -C banyand banyand-server` generates a basic `banyand-server`.
* `make -C banyand release` or `make -C banyand banyand-server-static` builds
out a static binary `banyand-server-static` for releasing.
-* `make -C banyand debug` gives a binary for debugging without the complier's
optimizations.
-* `make -C banyand debug-static` is a static binary for debugging.
-* `make -C bydbctl release` cross-builds several binaries for multi-platforms.
+* `make -C bydbctl bydbctl-cli` generates a basic `bydbctl-cli`.
+* `make -C bydbctl release` or `make -C banyand bydbctl-cli-static` builds out
a static binary `bydbctl-cli-static` for releasing.
Then users get binaries as below
``` shell
-$ ls banyand/build/bin
+ls banyand/build/bin/dev
banyand-server
banyand-server-static
-banyand-server-debug
-banyand-server-debug-static
-$ ls bydbctl/build/bin
-bydbctl bydbctl--darwin-amd64 bydbctl--darwin-arm64 bydbctl--linux-386
bydbctl--linux-amd64 bydbctl--linux-arm64 bydbctl--windows-386
bydbctl--windows-amd64
+ls bydbctl/build/bin/dev
+bydbctl-cli
+bydbctl-cli-static
```
> The build script now checks if the binary file exists before rebuilding. If
> you want to rebuild, please remove the binary file manually.
+
+### Cross-compile Binaries
+
+The build system supports cross-compiling binaries for different platforms.
For example, to build a Windows binary on a Linux machine, you can issue the
following command:
+
+```shell
+TARGET_OS=windows PLATFORMS=windows/amd64 make release
+```
+
+The `PLATFORMS` variable is a list of platforms separated by commas. The
`TARGET_OS` variable is the target operating system. You could specify several
platforms at once:
+
+```shell
+TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release
+TARGET_OS=darwin PLATFORMS=darwin/amd64,darwin/arm64 make release
+```
+
+The build system will generate binaries for each platform in the `build/bin`
directory.
+
+```shell
+darwin/amd64/banyand-server-static
+darwin/arm64/banyand-server-static
+linux/amd64/banyand-server-static
+linux/arm64/banyand-server-static
+windows/amd64/banyand-server-static
+
+darwin/amd64/bydbctl-cli-static
+darwin/arm64/bydbctl-cli-static
+linux/amd64/bydbctl-cli-static
+linux/arm64/bydbctl-cli-static
+windows/amd64/bydbctl-cli-static
+```
diff --git a/scripts/build/base.mk b/scripts/build/base.mk
index 58bd16f3..b96246d0 100644
--- a/scripts/build/base.mk
+++ b/scripts/build/base.mk
@@ -22,6 +22,13 @@ root_dir := $(mk_dir)../..
tool_bin := $(root_dir)/bin
tool_include := "$(root_dir)/include"
+# The main module path of a repository. For this repo, it is
github.com/tetrateio/tetrate.
+# MODULE_PATH is a variable used by some of the scripts in this subdirectory
to refer to the main
+# module name of this repository. This allows another project to import this
subdirectory (for
+# example, using a tool similar to: https://github.com/buildinspace/peru) and
override this
+# variable.
+MODULE_PATH ?= $(shell go mod edit -json | jq -r .Module.Path)
+
# Retrieve git versioning details so we can add to our binary assets
VERSION_PATH := github.com/apache/skywalking-banyandb/pkg/version
ifdef RELEASE_VERSION
@@ -34,6 +41,42 @@ endif
GO_LINK_VERSION := -X
${VERSION_PATH}.build=${VERSION_STRING}-${GIT_BRANCH_NAME}
+# Operating system name, such as 'Darwin'
+OS := $(shell uname)
+
+# TARGET_OS is the operating system to be used for building components for
local dev setups.
+ifeq ($(OS),Darwin)
+ TARGET_OS ?= darwin
+else ifeq ($(OS),Linux)
+ TARGET_OS ?= linux
+endif
+
+# Architecture, such as 'arm64'
+ARCH := $(shell uname -m)
+
+# Architecture to be used for building components for local dev setups.
+ifeq ($(ARCH),x86_64)
+ TARGET_ARCH ?= amd64
+else ifeq ($(ARCH),amd64)
+ TARGET_ARCH ?= amd64
+else ifeq ($(ARCH),arm64)
+ TARGET_ARCH ?= arm64
+endif
+
+# A comma-separated list of platforms for which the
+# docker images should be built. Builds for linux/<local-arch> for dev purposes
+# and multiple archs when run as part of the CI pipeline.
+PLATFORMS ?= $(if $(filter-out
$(CI),true),linux/$(TARGET_ARCH),linux/amd64,linux/arm64,windows/amd64)
+
+# Force using docker buildkit for all docker builds,
+# where not enabled by default.
+export DOCKER_BUILDKIT := 1
+
+# Avoid warning in Apple's new XCode 15 linker when using test -race.
+ifeq (Darwin,$(shell uname -s))
+LDFLAGS_COMMON ?= -extldflags=-Wl,-ld_classic
+endif
+
.PHONY: clean
clean:
git clean -Xdf
diff --git a/scripts/build/build.mk b/scripts/build/build.mk
index 47595d7c..29f8ec50 100644
--- a/scripts/build/build.mk
+++ b/scripts/build/build.mk
@@ -25,57 +25,41 @@ $(error The BINARIES variable should be set to the name
binaries to produce)
endif
STATIC_BINARIES ?= $(addsuffix -static,$(BINARIES))
-DEBUG_BINARIES ?= $(addsuffix -debug,$(BINARIES))
-DEBUG_STATIC_BINARIES ?= $(addsuffix -static,$(DEBUG_BINARIES))
BUILD_DIR ?= build/bin
+TARGET_OS ?= linux
+OS := ${TARGET_OS}
# Define SUB_DIR var if the project is not at root level project
-SOURCE_DIR := $(if $(SUB_DIR),$(SUB_DIR)/$(NAME),$(NAME))
-GOOS ?= linux
-GOARCH ?= amd64
+SOURCE_DIR ?= $(if $(SUB_DIR),$(SUB_DIR)/$(NAME),$(NAME))
+
+# Sentinel to guard against stale compiled binaries
+BUILD_LOCK := $(BUILD_DIR)/$(shell git rev-parse HEAD).lock
+
+# Build Go binaries for multiple architectures in the CI pipeline.
+GOBUILD_ARCHS := $(shell echo "${PLATFORMS}" | sed "s/linux\///g;
s/darwin\///g; s/windows\///g; s/\,/ /g" | tr ' ' '\n' | sort | uniq | tr '\n'
' ')
##@ Build targets
.PHONY: all
all: $(BINARIES) ## Build all the binaries
+BINARIES_GOBUILD_TARGET_PATTERN := $(BUILD_DIR)/dev/$(NAME)-%
+BINARIES_GOBUILD_TARGET := $(addprefix $(BUILD_DIR)/dev/,$(BINARIES))
+$(BINARIES): $(NAME)-%: $(BINARIES_GOBUILD_TARGET_PATTERN)
+$(BINARIES_GOBUILD_TARGET): $(BUILD_LOCK)
+ $(call set_build_package,$@,$@)
+ @echo "Building binary $@"
+ $(MAKE) prepare-build
+ $(call go_build_executable)
+ @echo "Done building $@"
-$(BINARIES): $(NAME)-%: $(BUILD_DIR)/$(NAME)-%
-$(addprefix $(BUILD_DIR)/,$(BINARIES)): $(BUILD_DIR)/$(NAME)-%:
- @echo "Building binary"
- GOOS=$(GOOS) GOARCH=$(GOARCH) go build -v -buildvcs=false --ldflags
'${GO_LINK_VERSION}' -tags "$(BUILD_TAGS)" -o $@
github.com/apache/skywalking-banyandb/$(SOURCE_DIR)/cmd/$*
- chmod +x $@
- @echo "Done building $(NAME) $*"
-
-.PHONY: debug
-debug: $(DEBUG_BINARIES) ## Build the debug binaries
-$(DEBUG_BINARIES): $(NAME)-%-debug: $(BUILD_DIR)/$(NAME)-%-debug
-$(addprefix $(BUILD_DIR)/,$(DEBUG_BINARIES)): $(BUILD_DIR)/$(NAME)-%-debug:
- @echo "Building debug binary"
- mkdir -p $(BUILD_DIR)
- GOOS=$(GOOS) GOARCH=$(GOARCH) go build -v -buildvcs=false --ldflags
'${GO_LINK_VERSION}' -tags "$(BUILD_TAGS)" -gcflags='all=-N -l' -o $@
github.com/apache/skywalking-banyandb/$(SOURCE_DIR)/cmd/$*
- chmod +x $@
- @echo "Done building debug $(NAME) $*"
-
-$(STATIC_BINARIES): $(NAME)-%-static: $(BUILD_DIR)/$(NAME)-%-static
-$(addprefix $(BUILD_DIR)/,$(STATIC_BINARIES)): $(BUILD_DIR)/$(NAME)-%-static:
- @echo "Building static binary"
- CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \
- -buildvcs=false \
- -a --ldflags '${GO_LINK_VERSION} -extldflags "-static"' -tags
"netgo $(BUILD_TAGS)" -installsuffix netgo \
- -o $(BUILD_DIR)/$(NAME)-$*-static
github.com/apache/skywalking-banyandb/$(SOURCE_DIR)/cmd/$*
- chmod +x $(BUILD_DIR)/$(NAME)-$*-static
- @echo "Done building static $(NAME) $*"
-
-.PHONY: debug-static
-debug-static: $(DEBUG_STATIC_BINARIES) ## Build the debug static binaries
-$(DEBUG_STATIC_BINARIES): $(NAME)-%-debug-static:
$(BUILD_DIR)/$(NAME)-%-debug-static
-$(addprefix $(BUILD_DIR)/,$(DEBUG_STATIC_BINARIES)):
$(BUILD_DIR)/$(NAME)-%-debug-static:
- @echo "Building debug static binary"
- CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \
- -buildvcs=false \
- -a --ldflags '${GO_LINK_VERSION} -extldflags "-static"' -tags
"netgo $(BUILD_TAGS)" -gcflags='all=-N -l' -installsuffix netgo \
- -o $(BUILD_DIR)/$(NAME)-$*-debug-static
github.com/apache/skywalking-banyandb/$(SOURCE_DIR)/cmd/$*
- chmod +x $(BUILD_DIR)/$(NAME)-$*-debug-static
- @echo "Done building debug static $(NAME) $*"
+STATIC_BINARIES_GOBUILD_TARGET_PATTERN := $(foreach
goarch,$(GOBUILD_ARCHS),$(BUILD_DIR)/$(OS)/$(goarch)/$(NAME)-%-static)
+STATIC_BINARIES_GOBUILD_TARGET := $(foreach
goarch,$(GOBUILD_ARCHS),$(addprefix
$(BUILD_DIR)/$(OS)/$(goarch)/,$(STATIC_BINARIES)))
+$(STATIC_BINARIES): $(NAME)-%-static: $(STATIC_BINARIES_GOBUILD_TARGET_PATTERN)
+$(STATIC_BINARIES_GOBUILD_TARGET): $(BUILD_DIR)/$(OS)/%-static: $(BUILD_LOCK)
+ $(call set_build_package,$*,$@)
+ @echo "Building static $*"
+ $(MAKE) prepare-build
+ $(call go_build_static_executable,,-s -w)
+ @echo "Done building static $*"
.PHONY: release
release: $(STATIC_BINARIES) ## Build the release binaries
@@ -83,3 +67,38 @@ release: $(STATIC_BINARIES) ## Build the release binaries
.PHONY: clean-build
clean-build: ## Clean all artifacts
rm -rf $(BUILD_DIR)
+
+# Helper target to prevent stale binaries from being packaged in Docker images.
+# This file is used as a sentinel for the last Git SHA of the built binaries.
If it does
+# not match the current Git SHA, it means the existing compiled binaries are
stale and should
+# be rebuilt again.
+$(BUILD_LOCK):
+ @echo "cleaning up stale build artifacts..."
+ $(MAKE) clean
+ mkdir -p $(BUILD_DIR)
+ touch $@
+
+# Helper functions for go build.
+# Takes in arguments 1. output binary name suffix 2. additional options to the
go build function.
+define go_build_executable
+ go build \
+ -v -ldflags '${GO_LINK_VERSION}' -tags "$(BUILD_TAGS)" $(1) \
+ -o $@ $(MODULE_PATH)/$(SOURCE_DIR)/cmd/$(build_package);
+ chmod +x $@;
+endef
+
+define go_build_static_executable
+ CGO_ENABLED=0 GOOS=${TARGET_OS} GOARCH=$(target_arch) go build \
+ $(fastbuild) -ldflags '${2} ${GO_LINK_VERSION} -extldflags
"-static"' \
+ -tags "netgo $(BUILD_TAGS)" -installsuffix netgo $(1)\
+ -o $@ $(MODULE_PATH)/$(SOURCE_DIR)/cmd/$(build_package);
+ chmod +x $@;
+endef
+
+
+# Sets the package name to be built as part of building $(NAME) project and
+# target architecture to be build for.
+define set_build_package
+ $(eval build_package := $(word 2,$(subst $(NAME)-, ,$1)))
+ $(eval target_arch := $(word 4,$(subst /, ,$2)))
+endef
diff --git a/scripts/build/docker.mk b/scripts/build/docker.mk
index 7c5d07e7..a3c6de5f 100644
--- a/scripts/build/docker.mk
+++ b/scripts/build/docker.mk
@@ -34,14 +34,13 @@ ifeq (true,$(CI))
DOCKER_BUILD_ARGS := $(DOCKER_BUILD_ARGS) --no-cache
endif
-docker: PLATFORMS =
docker: LOAD_OR_PUSH = --load
docker: DOCKER_TYPE = "Build"
-docker.push: PLATFORMS = --platform linux/amd64,linux/arm64,windows/amd64
docker.push: LOAD_OR_PUSH = --push
docker.push: DOCKER_TYPE = "Push"
docker docker.push:
- @echo "$(DOCKER_TYPE) $(IMG)"
- @time docker buildx build $(DOCKER_BUILD_ARGS) $(PLATFORMS)
$(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false ..
+ @echo "$(DOCKER_TYPE) $(IMG) with platform $(PLATFORMS)"
+ @pwd
+ time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS)
$(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false .
diff --git a/scripts/release.sh b/scripts/release.sh
index 21c12241..0182a2f7 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -36,22 +36,42 @@ binary(){
pushd ${tmpdir}
tar -xvf ${SOURCE_FILE}
make generate
- make release
+ TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make -C banyand release
bindir=./build
mkdir -p ${bindir}/bin
# Copy relevant files
- for module in "banyand" "bydbctl"
- do
- cp -Rfv ./${module}/build/bin/* ${bindir}/bin
- done
+ copy_binaries banyand
cp -Rfv ./CHANGES.md ${bindir}
cp -Rfv ./README.md ${bindir}
cp -Rfv ./dist/* ${bindir}
# Package
- tar -czf ${BUILDDIR}/skywalking-banyandb-${RELEASE_VERSION}-bin.tgz -C
${bindir} .
+ tar -czf ${BUILDDIR}/skywalking-banyandb-${RELEASE_VERSION}-banyand.tgz -C
${bindir} .
+
+ # Cross compile bydbctl
+ TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64,linux/386 make -C
bydbctl release
+ TARGET_OS=windows PLATFORMS=windows/amd64,windows/386 make -C bydbctl
release
+ TARGET_OS=darwin PLATFORMS=darwin/amd64,darwin/arm64 make -C bydbctl
release
+ rm -rf ${bindir}/bin
+ mkdir -p ${bindir}/bin
+ # Copy relevant files
+ copy_binaries bydbctl
+ # Package
+ tar -czf ${BUILDDIR}/skywalking-banyandb-${RELEASE_VERSION}-bydbctl.tgz -C
${bindir} .
popd
}
+copy_binaries() {
+ local module=$1
+ find ./${module}/build/bin -type f -not -name "*.lock" | while read -r
binary
+ do
+ # Extract os and arch from the path
+ os_arch=$(echo ${binary} | awk -F'/' '{print $(NF-2)"/"$(NF-1)}')
+ binary_name=$(basename ${binary})
+ binary_name=${binary_name%-*-*}
+ cp -Rfv ${binary} ${bindir}/bin/${binary_name}-${os_arch//\//-}
+ done
+}
+
source(){
# Package
mkdir -p ${BUILDDIR}
@@ -60,13 +80,11 @@ source(){
echo "RELEASE_VERSION=${RELEASE_VERSION}" > .env
tar \
--exclude=".DS_Store" \
- --exclude=".git" \
--exclude=".github" \
--exclude=".gitignore" \
--exclude=".asf.yaml" \
--exclude=".idea" \
--exclude=".vscode" \
- --exclude="./build" \
--exclude="bin" \
-czf ${SOURCE_FILE} \
.
