This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 28df30ed0 chore(go-client-ci): build the server instead of downloading
it from external site (#1790)
28df30ed0 is described below
commit 28df30ed08d9f022543a953c196b3f4805ace732
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Dec 26 15:39:02 2023 +0800
chore(go-client-ci): build the server instead of downloading it from
external site (#1790)
https://github.com/apache/incubator-pegasus/issues/1779
- separate format and lint jobs
- build Pegasus server binaries in the CI procedure instead of downloading
it from any site
- fix lint and test failures
- improve readme
---
.github/workflows/lint_and_test_go-client.yml | 116 +++++++++++++++++++++-----
go-client/README.md | 24 +++++-
go-client/admin/client_test.go | 6 +-
go-client/example/main.go | 2 +-
go-client/generator/main.go | 70 +++++++---------
go-client/idl/base/dsn_err_string.go | 1 -
go-client/session/admin_rpc_types.go | 35 ++++----
go-client/session/radmin_rpc_types.go | 35 ++++----
go-client/test.sh | 46 ----------
9 files changed, 184 insertions(+), 151 deletions(-)
diff --git a/.github/workflows/lint_and_test_go-client.yml
b/.github/workflows/lint_and_test_go-client.yml
index 011ae97c3..79d9e12b0 100644
--- a/.github/workflows/lint_and_test_go-client.yml
+++ b/.github/workflows/lint_and_test_go-client.yml
@@ -35,31 +35,103 @@ on:
# for manually triggering workflow
workflow_dispatch:
+env:
+ ARTIFACT_NAME: release_for_go_client
+
# workflow tasks
jobs:
- build:
- name: Test and Lint
+ format:
+ name: Format
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Setup Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.18
+ - name: Format
+ working-directory: ./go-client
+ run: |
+ gofmt -d .
+ test -z "$(gofmt -d .)"
+
+ lint:
+ name: Lint
+ needs: format
+ # go-client imports thrift package of 0.13.0, so we must use
thrift-compiler 0.13.0
+ # to generate code as well. The thrift-compiler version on ubuntu-20.04 is
0.13.0
runs-on: ubuntu-20.04
steps:
- - name: Install thrift
- # go-client imports thrift package of 0.13.0, so we must use
thrift-compiler 0.13.0
- # to generate code as well. The thrift-compiler version on ubuntu-20.04
is 0.13.0
- run: sudo apt-get install -y thrift-compiler
- - name: Checkout
- uses: actions/checkout@v3
- - name: Set up Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.18
- - name: Run Test
- run: ./test.sh
- working-directory: ./go-client
- - uses: codecov/codecov-action@v2
- with:
+ - name: Install thrift
+ run: sudo apt-get install -y thrift-compiler
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Setup Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.18
+ - name: Build
working-directory: ./go-client
- # because some files are generated after building, so lint after that at
last
- - name: Lint
- uses: golangci/golangci-lint-action@v3
- with:
- version: v1.55.2
+ run: make build
+ # because some files are generated after building, so lint after the
build step
+ - name: Lint
+ uses: golangci/golangci-lint-action@v3
+ with:
+ version: v1.55.2
+ working-directory: ./go-client
+
+ build_server:
+ name: Build server
+ needs: lint
+ runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
+ BUILD_OPTIONS: -t release
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
+ steps:
+ - uses: actions/checkout@v3
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
+
+ test_go_client:
+ name: Test Go client
+ needs: build_server
+ runs-on: ubuntu-latest
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
+ steps:
+ # go-client imports thrift package of 0.13.0, so we must use
thrift-compiler 0.13.0
+ # to generate code as well. The thrift-compiler version on ubuntu-22.04
is 0.16.0, so
+ # build and install thrift-compiler 0.13.0 manually.
+ - name: Install thrift
+ run: |
+ export THRIFT_VERSION=0.13.0
+ wget --progress=dot:giga
https://github.com/apache/thrift/archive/refs/tags/v${THRIFT_VERSION}.tar.gz
+ tar -xzf v${THRIFT_VERSION}.tar.gz
+ cd thrift-${THRIFT_VERSION}
+ ./bootstrap.sh
+ ./configure --enable-libs=no
+ make -j $(nproc)
+ make install
+ cd - && rm -rf thrift-${THRIFT_VERSION} v${THRIFT_VERSION}.tar.gz
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Set up Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.18
+ - uses: "./.github/actions/download_artifact"
+ - name: Start Pegasus cluster
+ run: |
+ export
LD_LIBRARY_PATH=$(pwd)/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
+ ulimit -s unlimited
+ ./run.sh start_onebox
+ - name: Run Go client tests
working-directory: ./go-client
+ run: |
+ GO111MODULE=on make build
+ ./bin/echo > /dev/null 2>&1 &
+ GO111MODULE=on make ci
diff --git a/go-client/README.md b/go-client/README.md
index 2247c041f..65fc86e48 100644
--- a/go-client/README.md
+++ b/go-client/README.md
@@ -22,11 +22,31 @@ under the License.
## Requirement
-Go1.18+.
+- Go1.18+
+- thrift 0.13
+
+## Development
+
+Build the pegasus-go-client:
+```bash
+make build
+```
+
+Format the code:
+```bash
+make fmt
+```
+
+It requires the Pegasus [onebox](https://pegasus.apache.org/overview/onebox/)
has been started.
+
+Then run tests:
+```bash
+make ci
+```
## Logging
-By default pegasus-go-client logs to "./pegasus.log" on where your application
runs.
+By default, pegasus-go-client logs to "./pegasus.log" on where your
application runs.
You can customize the logging rules as follows:
```go
diff --git a/go-client/admin/client_test.go b/go-client/admin/client_test.go
index bd2b7b63e..3d6ee3ddf 100644
--- a/go-client/admin/client_test.go
+++ b/go-client/admin/client_test.go
@@ -78,20 +78,20 @@ func TestAdmin_CreateTableMustAvailable(t *testing.T) {
const tableName = "admin_table_test"
c := NewClient(Config{
- MetaServers: []string{"0.0.0.0:34601"},
+ MetaServers: []string{"0.0.0.0:34601", "0.0.0.0:34602",
"0.0.0.0:34603"},
})
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
- err := c.CreateTable(context.Background(), tableName, 1)
+ err := c.CreateTable(context.Background(), tableName, 8)
if !assert.NoError(t, err) {
assert.Fail(t, err.Error())
}
// ensures the created table must be available for read and write
rwClient := pegasus.NewClient(pegasus.Config{
- MetaServers: []string{"0.0.0.0:34601"},
+ MetaServers: []string{"0.0.0.0:34601", "0.0.0.0:34602",
"0.0.0.0:34603"},
})
defer func() {
err = rwClient.Close()
diff --git a/go-client/example/main.go b/go-client/example/main.go
index 2d679f288..cab1c462e 100644
--- a/go-client/example/main.go
+++ b/go-client/example/main.go
@@ -65,7 +65,7 @@ func main() {
for t := 0; t < 10; t++ {
var sortKeys [][]byte
for i := 0; i < 10; i++ {
- sortKeys = append(sortKeys, []byte("sort"+string(i)))
+ sortKeys = append(sortKeys,
[]byte("sort"+fmt.Sprint(i)))
}
for i := 0; i < 10; i++ {
err = tb.Set(context.Background(), []byte("hash"),
sortKeys[i], value)
diff --git a/go-client/generator/main.go b/go-client/generator/main.go
index 4be929ee0..f88caf59c 100644
--- a/go-client/generator/main.go
+++ b/go-client/generator/main.go
@@ -62,6 +62,32 @@ func generateAdminRPC(rpcCode, rpcName, reqName, respName
string) string {
return s
}
+func generateHeader() {
+ fmt.Print(`/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Code generated by "generator -i=admin.csv > admin_rpc_types.go"; DO NOT
EDIT.
+package session
+
+`)
+}
+
func generateAllAdminRPC() {
content, err := ioutil.ReadFile(inputFilePath)
if err != nil {
@@ -77,28 +103,8 @@ func generateAllAdminRPC() {
os.Exit(1)
}
- fmt.Print(`/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/`)
+ generateHeader()
fmt.Print(`
-// Code generated by "generator -i=admin.csv > admin_rpc_types.go"; DO NOT
EDIT.
-package session
-
import (
"context"
"fmt"
@@ -153,28 +159,8 @@ func generateAllRAdminRPC() {
os.Exit(1)
}
- fmt.Print(`/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/`)
+ generateHeader()
fmt.Print(`
-// Code generated by "generator -i=admin.csv > admin_rpc_types.go"; DO NOT
EDIT.
-package session
-
import (
"context"
"fmt"
diff --git a/go-client/idl/base/dsn_err_string.go
b/go-client/idl/base/dsn_err_string.go
index b68183c7e..4a41dd95d 100644
--- a/go-client/idl/base/dsn_err_string.go
+++ b/go-client/idl/base/dsn_err_string.go
@@ -1,6 +1,5 @@
// Code generated by "enumer -type=DsnErrCode -output=dsn_err_string.go"; DO
NOT EDIT.
-//
package base
import (
diff --git a/go-client/session/admin_rpc_types.go
b/go-client/session/admin_rpc_types.go
index 35bfcfec1..05526c987 100644
--- a/go-client/session/admin_rpc_types.go
+++ b/go-client/session/admin_rpc_types.go
@@ -1,21 +1,22 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
// Code generated by "generator -i=admin.csv > admin_rpc_types.go"; DO NOT
EDIT.
package session
diff --git a/go-client/session/radmin_rpc_types.go
b/go-client/session/radmin_rpc_types.go
index 55e840f75..7ba9100da 100644
--- a/go-client/session/radmin_rpc_types.go
+++ b/go-client/session/radmin_rpc_types.go
@@ -1,21 +1,22 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
// Code generated by "generator -i=admin.csv > admin_rpc_types.go"; DO NOT
EDIT.
package session
diff --git a/go-client/test.sh b/go-client/test.sh
deleted file mode 100755
index 3cd223d42..000000000
--- a/go-client/test.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
-PROJECT_DIR=$(dirname "${SCRIPT_DIR}")
-cd "${PROJECT_DIR}" || exit 1
-
-PEGASUS_PKG="pegasus-tools-2.0.0-5d969e8-glibc2.12-release"
-PEGASUS_PKG_URL="https://github.com/apache/incubator-pegasus/releases/download/v2.0.0/pegasus-tools-2.0.0-5d969e8-glibc2.12-release.tar.gz";
-
-# start pegasus onebox environment
-if [ ! -f $PEGASUS_PKG.tar.gz ]; then
- wget --quiet $PEGASUS_PKG_URL
- tar xf $PEGASUS_PKG.tar.gz
-fi
-cd $PEGASUS_PKG
-
-./run.sh start_onebox -m 2 -r 3 -w
-cd ../
-
-GO111MODULE=on make build
-./bin/echo > /dev/null 2>&1 & # run echoserver in the background, drop its
stderr/stdout
-
-if ! GO111MODULE=on time make ci
-then
- cd $PEGASUS_PKG
- ./run.sh list_onebox
- exit 1
-fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
