This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new 0d0ba8e4 Test publishing Docker images in CI (#513)
0d0ba8e4 is described below
commit 0d0ba8e4683c210b1a05ace9640239e715c73b81
Author: Andy Grove <[email protected]>
AuthorDate: Thu Nov 17 12:21:14 2022 -0700
Test publishing Docker images in CI (#513)
* Prep work for running benchmark queries in CI and checking for expected
results
* standalone dockerfile
* revert accidental change
* publish docker images
* Add to build scripts
* install wget
* change namespace
* Revert "change namespace"
This reverts commit d3b1b2f820b1dc93593442cfd5740c6424d4e1ef.
Co-authored-by: Brent Gardner <[email protected]>
---
.github/workflows/rust.yml | 75 +++++++++++++++++++---
dev/build-ballista-docker.sh | 1 +
dev/docker/ballista-standalone.Dockerfile | 51 +++++++++++++++
.../docker/standalone-entrypoint.sh | 33 ++++------
4 files changed, 131 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 63eb09fc..3f595114 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -57,18 +57,43 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- - name: Build workspace in debug mode
- run: |
- cargo build
- env:
- CARGO_HOME: "/github/home/.cargo"
- CARGO_TARGET_DIR: "/github/home/target/debug"
- name: Build workspace in release mode
run: |
- cargo check --release
+ cargo build --release
+ ls -l /github/home/target/release
env:
CARGO_HOME: "/github/home/.cargo"
- CARGO_TARGET_DIR: "/github/home/target/release"
+ CARGO_TARGET_DIR: "/github/home/target"
+ - name: Save artifacts
+ uses: actions/upload-artifact@v2
+ with:
+ name: rust-artifacts
+ path: |
+ /github/home/target/release/ballista-scheduler
+ /github/home/target/release/ballista-executor
+
+ react-build:
+ name: React build
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install dependencies
+ run: |
+ sudo apt-get -qq update
+ sudo apt-get -qq install -y nodejs npm
+ npm install -g yarn
+ - name: Run yarn build
+ run: |
+ cd ballista/scheduler/ui
+ pwd
+ yarn install
+ yarn build
+ - name: Save artifacts
+ uses: actions/upload-artifact@v2
+ with:
+ name: react-artifacts
+ path: |
+ ballista/scheduler/ui/build
# test the crate
linux-test:
@@ -257,6 +282,40 @@ jobs:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
+ docker:
+ name: Docker
+ needs: [linux-build-lib, react-build]
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - uses: actions/checkout@v3
+ - name: Restore rust artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: rust-artifacts
+ path: /home/runner/work/arrow-ballista/arrow-ballista/target/release
+ - name: Restore react artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: react-artifacts
+ path:
/home/runner/work/arrow-ballista/arrow-ballista/ballista/scheduler/ui/build
+ - name: Build and push Docker image
+ run: |
+ echo "github user is $DOCKER_USER"
+ export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1
--pretty='%h') || echo '0.10.0-test')"
+ if [[ $DOCKER_TAG =~ ^[0-9\.]+$ ]]
+ then
+ echo "publishing docker tag $DOCKER_TAG"
+ fi
+ docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS"
+ docker build -t ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
-f dev/docker/ballista-standalone.Dockerfile .
+ docker push ghcr.io/apache/arrow-ballista-standalone:$DOCKER_TAG
+ env:
+ DOCKER_USER: ${{ github.actor }}
+ DOCKER_PASS: ${{ secrets.GITHUB_TOKEN }}
+
clippy:
name: Clippy
needs: [linux-build-lib]
diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh
index 97288431..cead5e84 100755
--- a/dev/build-ballista-docker.sh
+++ b/dev/build-ballista-docker.sh
@@ -28,6 +28,7 @@ docker run -v $(pwd):/home/builder/workspace --env
RELEASE_FLAG=$RELEASE_FLAG ba
docker-compose build
. ./dev/build-set-env.sh
+docker build -t "apache/arrow-ballista-standalone:$BALLISTA_VERSION" -f
dev/docker/ballista-standalone.Dockerfile .
docker tag ballista-executor "apache/arrow-ballista-executor:$BALLISTA_VERSION"
docker tag ballista-scheduler
"apache/arrow-ballista-scheduler:$BALLISTA_VERSION"
docker tag ballista-benchmarks
"apache/arrow-ballista-benchmarks:$BALLISTA_VERSION"
diff --git a/dev/docker/ballista-standalone.Dockerfile
b/dev/docker/ballista-standalone.Dockerfile
new file mode 100644
index 00000000..caa877c4
--- /dev/null
+++ b/dev/docker/ballista-standalone.Dockerfile
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:22.04
+
+ARG RELEASE_FLAG=release
+
+ENV RELEASE_FLAG=${RELEASE_FLAG}
+ENV RUST_LOG=info
+ENV RUST_BACKTRACE=full
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -qq update && apt-get install -qq -y nginx netcat wget
+
+COPY target/$RELEASE_FLAG/ballista-scheduler /root/ballista-scheduler
+COPY target/$RELEASE_FLAG/ballista-executor /root/ballista-executor
+
+# populate some sample data for ListingSchemaProvider
+RUN mkdir -p /data && \
+ wget -q
https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet
-P /data/
+ENV DATAFUSION_CATALOG_LOCATION=/data
+ENV DATAFUSION_CATALOG_TYPE=csv
+
+COPY ballista/scheduler/ui/build /var/www/html
+COPY dev/docker/nginx.conf /etc/nginx/sites-enabled/default
+
+# Expose Ballista Scheduler web UI port
+EXPOSE 80
+
+# Expose Ballista Scheduler gRPC port
+EXPOSE 50050
+
+# Expose Ballista Executor gRPC port
+EXPOSE 50051
+
+COPY dev/docker/standalone-entrypoint.sh /root/standalone-entrypoint.sh
+ENTRYPOINT ["/root/standalone-entrypoint.sh"]
diff --git a/.github/workflows/typescript.yml
b/dev/docker/standalone-entrypoint.sh
similarity index 62%
rename from .github/workflows/typescript.yml
rename to dev/docker/standalone-entrypoint.sh
index fad72995..06d3dbf0 100644
--- a/.github/workflows/typescript.yml
+++ b/dev/docker/standalone-entrypoint.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,27 +17,16 @@
# specific language governing permissions and limitations
# under the License.
-name: TypeScript Release Build
+set -e
-on:
- # always trigger
- push:
- pull_request:
+echo "Starting nginx web UI..."
+nohup nginx -g "daemon off;" &
-jobs:
- yarn-build:
- name: Yarn build
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - name: Install dependencies
- run: |
- sudo apt-get update
- sudo apt-get install -y nodejs npm
- npm install -g yarn
+echo "Starting for scheduler..."
+/root/ballista-scheduler &
+while ! nc -z 127.0.0.1 50050; do
+ sleep 1
+done
- - name: Run yarn build
- run: |
- cd ballista/scheduler/ui
- yarn install
- yarn build
\ No newline at end of file
+echo "Starting executor"
+/root/ballista-executor
