This is an automated email from the ASF dual-hosted git repository. xuba pushed a commit to branch v0.7.x-test-front in repository https://gitbox.apache.org/repos/asf/amoro.git
commit d31d9cad8164489a072682735bb67dcccf30e21b Author: Xavier Bai <[email protected]> AuthorDate: Mon Jan 29 14:59:40 2024 +0800 [WBXWAP-126939] add CI Jenkinsfile --- Jenkinsfile | 102 ++++++++++++++++++++++++++++++++++ docker/wap/Dockerfile | 65 ++++++++++++++++++++++ docker/wap/entrypoint.sh | 63 +++++++++++++++++++++ docker/wap/prometheus-jmx-config.yaml | 20 +++++++ manifest.yaml | 55 ++++++++++++++++++ 5 files changed, 305 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..9e8312bba --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@Library(['sapPipeline@enhancemnent','ciHelper@master']) _ + +def buildArgs1 = [:] + +def amoro_version() { + def version = sh(script: "cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print \$3}'", returnStdout: true).trim() + return version +} + +def imageTag() { + gitCommitHash = sh (script: "git log -n 1 --pretty=format:'%h'", returnStdout: true).trim() + return "${amoro_version()}_" + gitCommitHash +} + +def workDir() { + return "/jenkins/workspace/workspace/Amoro_Pipeline_${env.BRANCH_NAME}" +} + +def dockerQAUrl() { + return "registry-qa.webex.com/wap-dataprocessor/amoro:${imageTag()}" +} + +def dockerPRODUrl() { + return "registry-rmc.webex.com/wap-dataprocessor/amoro:${imageTag()}" +} + +def amoro_metaBody = { + artifact_id = imageTag() + image_tag = imageTag() + describe = "Amoro image build ci pipeline" +} + +pipeline { + agent { label 'SJC' } + tools { + maven 'Maven 3.6.1' + } + stages { + stage('Checkout') { + steps { + checkout scm + } + } + stage('Maven install') { + steps { + script { + sh("mvn clean install -DskipTests -am -e -pl ams/dist") + } + } + } + stage('Build images') { + steps { + script { + sh("mv ${workDir()}/ams/dist/target/amoro-${amoro_version()}-bin.zip ${workDir()}/docker/wap/") + } + script { + sh("docker build --build-arg AMORO_VERSION=${amoro_version()} ${workDir()}/docker/wap -t ${dockerQAUrl()}") + } + } + } + stage('Push Amoro PROD image') { + steps { + script { + sh("docker push ${dockerQAUrl()}") + } + } + } + stage('Push ECR image') { + steps { + script { + buildArgs1 = [component: "arctic", tag: imageTag(), metadata: amoro_metaBody, dockerArgs: "AMORO_VERSION=${amoro_version()}"] + } + script { + buildCI(this, buildArgs1) + } + } + } + stage('Clean up') { + steps { + deleteDir() + } + } + } +} diff --git a/docker/wap/Dockerfile b/docker/wap/Dockerfile new file mode 100644 index 000000000..75b58589d --- /dev/null +++ b/docker/wap/Dockerfile @@ -0,0 +1,65 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=linux/amd64 eclipse-temurin:8-jdk-jammy as builder + +# Add the entire project to the build container, unzip it, +# and remove flink-optimizer to reduce the container size. + +ADD . /workspace/amoro +WORKDIR /workspace/amoro +ARG AMORO_VERSION + +RUN apt-get -qq -y update \ + && apt-get -qq -y upgrade \ + && apt-get install -qq -y vim \ + && apt-get install -qq -y unzip + +RUN cp amoro-${AMORO_VERSION}-bin.zip /usr/local \ + && unzip /usr/local/amoro-${AMORO_VERSION}-bin.zip -d /usr/local \ + && rm /usr/local/amoro-${AMORO_VERSION}/plugin/optimizer/flink -rf \ + && mv /usr/local/amoro-${AMORO_VERSION} /usr/local/amoro \ + && rm -rf /workspace/amoro + + +FROM --platform=linux/amd64 eclipse-temurin:8-jdk-jammy + +ARG USER_UID=10000 +ARG PROMETHEUS_AGENT_VERSION=0.18.0 + +ENV AMORO_HOME /usr/local/amoro +ENV AMORO_CONF_DIR ${AMORO_HOME}/conf +ENV LOG_LEVEL info +EXPOSE 1630 9093 8088 + +COPY entrypoint.sh / +COPY --from=builder "/usr/local/amoro" "/usr/local/amoro" + +WORKDIR ${AMORO_HOME} + +COPY prometheus-jmx-config.yaml conf/ +RUN curl -L -o lib/mysql-connector-java-8.0.30.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar \ + && curl -L -o plugin/jmx_prometheus_javaagent-$PROMETHEUS_AGENT_VERSION.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/$PROMETHEUS_AGENT_VERSION/jmx_prometheus_javaagent-$PROMETHEUS_AGENT_VERSION.jar \ + && chmod +x /entrypoint.sh \ + && adduser --uid $USER_UID --disabled-password --gecos "" amoro \ + && chown -R amoro:amoro ${AMORO_HOME} \ + && chown -R amoro:amoro /entrypoint.sh + +USER amoro + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["ams"] \ No newline at end of file diff --git a/docker/wap/entrypoint.sh b/docker/wap/entrypoint.sh new file mode 100644 index 000000000..923084fcc --- /dev/null +++ b/docker/wap/entrypoint.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +args=("$@") + +if [ -n "$LOG_LEVEL" ]; then + # set log-level to console-log-level + export CONSOLE_LOG_LEVEL=$LOG_LEVEL +fi + +configure_jvm_options() { + JVM_PROPERTIES_FILE="${AMORO_CONF_DIR}/jvm.properties" + for option in $(printenv | perl -sne 'print "$1 " if m/^JVM_(.+?)=.*/' ); do + key=$(echo "$option" | perl -pe 's/__/-/g; s/_/./g;') + var="JVM_$option" + value=${!var} + declare -l lowerKey=$key + if grep "$lowerKey" "$JVM_PROPERTIES_FILE" >/dev/null + then + sed -i "s/$lowerKey=.*$/$lowerKey=\"$value\"/g" "$JVM_PROPERTIES_FILE" + else + sed -i '$a/'"$lowerKey=\"$value\"" "$JVM_PROPERTIES_FILE" + fi + done +} + + +configure_jvm_options + + +if [ $1 == "help" ]; then + printf "Usage: $(basename $0) [ams|optimizer] [args]\n" + printf " Or: $(basename $0) help \n\n" + exit 0 +elif [ "$1" == "ams" ]; then + args=("${args[@]:1}") + echo "Start Amoro Management Service" + exec "$AMORO_HOME/bin/ams.sh" "start-foreground" +elif [ "$1" == "optimizer" ]; then + args=("${args[@]:1}") + echo "Start Amoro Optimizer" + exec "$AMORO_HOME/bin/optimizer.sh " "${args[@]}" +fi + +# Running command in pass-through mode +exec "${args[@]}" \ No newline at end of file diff --git a/docker/wap/prometheus-jmx-config.yaml b/docker/wap/prometheus-jmx-config.yaml new file mode 100644 index 000000000..cadb5ce5b --- /dev/null +++ b/docker/wap/prometheus-jmx-config.yaml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ssl: false +lowercaseOutputName: false +lowercaseOutputLabelNames: false \ No newline at end of file diff --git a/manifest.yaml b/manifest.yaml new file mode 100644 index 000000000..abc881d31 --- /dev/null +++ b/manifest.yaml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +- name: reference_app + repository: https://sqbu-github.cisco.com/SAP/amoro.git + graceful_shutdown: false + metadata: + - url: https://metadata.prod.infra.webex.com + cert-id: wap-metadata-service-prod-client-cert + key-id: wap-metadata-service-prod-client-key + registries: + - name: registry-qa.webex.com + namespace: wap-dataprocessor + component: arctic + service_group: webex-wap + operation_type: QA-Done + type: docker + jenkins-id: 07093dbc-68c7-42d6-b422-4f8b522922f1 + publish: true + publish-always: true + docker: + - dockerfile: docker/wap/Dockerfile + lint: + enable: false + scan: + enable: false + build: true + security: + - policy-repo: https://sqbu-github.cisco.com/WebexPlatform/wbx3-policies.git + bundle: meetpaas_default_bundle.json + jenkins-id: 07093dbc-68c7-42d6-b422-4f8b522922f1 + threat-levels: + - Critical + - High + gate: false + contact: + - mailer: + - [email protected] + teams-space-id: 453d42c0-1202-11ec-bb17-73b64b411a4a + teams-jenkins-id: bot_token
