This is an automated email from the ASF dual-hosted git repository. okislal pushed a commit to branch madlib2-master in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 03fba10fc0d115afcf25c00e1b97ef747676396a Author: Orhan Kislal <okis...@apache.org> AuthorDate: Wed Mar 29 23:01:58 2023 +0300 Update jenkins to use PG13 The current jenkins build is failing because of a server crash. Added a FIXME in the build script to look at this problem and find an efficient way to debug in a different JIRA. --- src/bin/madpack | 2 +- tool/docker/base/Dockerfile_postgres_13_Jenkins | 40 +++++++++++++++++++++++++ tool/jenkins/jenkins_build.sh | 21 +++++++------ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/bin/madpack b/src/bin/madpack index 0507fbc2..66fea155 100755 --- a/src/bin/madpack +++ b/src/bin/madpack @@ -13,7 +13,7 @@ # 2. Pass all arguments to ../madpack/madpack.py PYTHON_PREFIX="python" -PYTHON_VERSIONS="2.7 2.6" +PYTHON_VERSIONS="2.7 2.6 3" # create absolute path to madpack.py pushd `dirname $0` > /dev/null diff --git a/tool/docker/base/Dockerfile_postgres_13_Jenkins b/tool/docker/base/Dockerfile_postgres_13_Jenkins new file mode 100644 index 00000000..59ecd695 --- /dev/null +++ b/tool/docker/base/Dockerfile_postgres_13_Jenkins @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM postgres:13 + +### Get postgres specific add-ons +RUN apt-get update && apt-get install -y wget \ + build-essential \ + postgresql-server-dev-13 \ + postgresql-plpython3-13 \ + openssl \ + libssl-dev \ + libboost-all-dev \ + m4 \ + rpm \ + python3-pip \ + python3-dev \ + build-essential \ + cmake \ + libspatialindex-dev + +RUN python3 -m pip install tensorflow dill rtree xgboost + +## To build an image from this docker file, from madlib folder, run: +# docker build -t madlib/postgres_13:jenkins -f tool/docker/base/Dockerfile_postgres_13_Jenkins . diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh index 904a102c..f1df043f 100755 --- a/tool/jenkins/jenkins_build.sh +++ b/tool/jenkins/jenkins_build.sh @@ -20,6 +20,9 @@ workdir=`pwd` user_name=`whoami` +# FIXME: Jenkins build is not working, even though PG13 compiles on local Mac. +# Need to find an efficient way to debug since it has a server crash. + echo "======================================================================" echo "Build user: $user_name" echo "Work directory: $workdir" @@ -43,14 +46,14 @@ docker rm madlib echo "Creating docker container" # Pull down the base docker images -echo "docker pull madlib/postgres_11:jenkins" -docker pull madlib/postgres_11:jenkins +echo "docker pull madlib/postgres_13:jenkins" +docker pull madlib/postgres_13:jenkins # Launch docker container with volume mounted from workdir echo "-------------------------------" cat <<EOF -docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_11:jenkins | tee logs/docker_setup.log +docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_13:jenkins | tee logs/docker_setup.log EOF -docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_11:jenkins | tee logs/docker_setup.log +docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_13:jenkins | tee logs/docker_setup.log echo "-------------------------------" ## This sleep is required since it takes a couple of seconds for the docker @@ -60,9 +63,9 @@ sleep 15 echo "---------- Install pip, and mock -----------" # cmake, make, make install, and make package cat <<EOF -docker exec madlib bash -c 'apt-get update; apt-get install -y python-pip; pip install mock pandas numpy xgboost scikit-learn' | tee $workdir/logs/madlib_compile.log +docker exec madlib bash -c 'apt-get update; apt-get install -y python3-pip; pip install mock pandas numpy xgboost scikit-learn yaml' | tee $workdir/logs/madlib_compile.log EOF -docker exec madlib bash -c 'apt-get update; apt-get install -y python-pip; pip install mock pandas numpy xgboost scikit-learn' | tee $workdir/logs/madlib_compile.log +docker exec madlib bash -c 'apt-get update; apt-get install -y python3-pip; pip install mock pandas numpy xgboost scikit-learn yaml' | tee $workdir/logs/madlib_compile.log echo "---------- Building package -----------" # cmake, make, make install, and make package @@ -86,7 +89,7 @@ EOF docker exec madlib bash -c 'mkdir -p /tmp' docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/modules/deep_learning/test' -docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/11/modules/deep_learning/test' +docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/13/modules/deep_learning/test' # Run dev check docker exec madlib bash -c '/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check' | tee $workdir/logs/madlib_dev_check.log # Run unit tests, and append output to dev_check's log file @@ -108,6 +111,6 @@ echo "-------------------------------" # convert dev-check test results to junit format for reporting cat <<EOF -python tool/jenkins/junit_export.py $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml +python3 tool/jenkins/junit_export.py $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml EOF -python tool/jenkins/junit_export.py $workdir $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml +python3 tool/jenkins/junit_export.py $workdir $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml