[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 72a6e1e8bfdb606a6b6acee0abef43fdc13d5820 Author: Magnus Granberg gentoo org> AuthorDate: Wed Jun 7 21:33:45 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Jun 7 21:33:45 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=72a6e1e8 Add support for xz compression Signed-off-by: Magnus Granberg gentoo.org> docker/GentooBuildbotWorkerBuild.Dockerfile | 5 - docker/GentooBuildbotWorkerLog.Dockerfile | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/GentooBuildbotWorkerBuild.Dockerfile b/docker/GentooBuildbotWorkerBuild.Dockerfile index 23fc3f0..e700203 100644 --- a/docker/GentooBuildbotWorkerBuild.Dockerfile +++ b/docker/GentooBuildbotWorkerBuild.Dockerfile @@ -35,6 +35,8 @@ RUN echo -e "[binhost]\npriority = \nsync-uri = ${BINHOSTURL}/${PROJECTUUID} && echo 'USE="X elogind"' | >> /etc/portage/make.conf\ # && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >> /etc/portage/make.conf\ && echo 'ACCEPT_LICENSE="*"'\ + && echo 'PORTAGE_COMPRESS="xz"'\ + && echo 'BINPKG_COMPRESS="xz"'\ #&& echo 'dev-vcs/git -webdev -gnome-keyring' | cat >> /etc/portage/package.use/git\ #&& echo 'dev-util/buildbot-worker' | cat >> /etc/portage/package.accept_keywords/buildbot\ #&& echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\ @@ -62,7 +64,8 @@ RUN echo 'app-text/ansifilter' | cat >> /var/lib/portage/world\ && echo 'dev-lang/rust-bin' | cat >> /var/lib/portage/world\ && echo 'app-admin/eclean-kernel' | cat >> /var/lib/portage/world\ && echo 'app-portage/gentoolkit' | cat >> /var/lib/portage/world\ - && echo 'app-editors/nano' | cat >> /var/lib/portage/world + && echo 'app-editors/nano' | cat >> /var/lib/portage/world\ + && echo 'app-arch/xz-utils' | cat >> /var/lib/portage/world # Run emerge update and install buildbot-worker # update perl diff --git a/docker/GentooBuildbotWorkerLog.Dockerfile b/docker/GentooBuildbotWorkerLog.Dockerfile index 492be36..0164b94 100644 --- a/docker/GentooBuildbotWorkerLog.Dockerfile +++ b/docker/GentooBuildbotWorkerLog.Dockerfile @@ -18,7 +18,7 @@ RUN echo -e "[binhost]\npriority = \nsync-uri = https://gentoo.osuosl.org/ex && echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\ && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\ && echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\ - && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin dev-vcs/git + && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin dev-vcs/git utils-xz # We don't need the gentoo repo any longer RUN rm -R /var/db/repos/gentoo
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 0d402a82fd38e399a0b8185734cd796d0215b5fc Author: Magnus Granberg gentoo org> AuthorDate: Mon May 8 20:14:25 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Mon May 8 20:14:25 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d402a82 Use user buildbot for log image Signed-off-by: Magnus Granberg gentoo.org> docker/GentooBuildbotWorkerLog.Dockerfile | 26 -- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docker/GentooBuildbotWorkerLog.Dockerfile b/docker/GentooBuildbotWorkerLog.Dockerfile index a7c7af1..492be36 100644 --- a/docker/GentooBuildbotWorkerLog.Dockerfile +++ b/docker/GentooBuildbotWorkerLog.Dockerfile @@ -1,3 +1,6 @@ +# This docker file is use for the log parser and ebuild parser image +# Use stable arch and gentoo's experimental binhost + # name the portage image FROM gentoo/portage:latest as portage @@ -7,24 +10,27 @@ FROM gentoo/stage3:latest # copy the entire portage volume in COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo -# Setup portage -# emerge needed deps buildbot-worker, psycopg, git and sqlalchemy -# get the needed buildbot-worker config +# Setup portage and emerge needed deps buildbot-worker, psycopg, git and sqlalchemy RUN echo -e "[binhost]\npriority = \nsync-uri = https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n; | cat >> /etc/portage/binrepos.conf\ && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y --autounmask-write --autounmask-continue --autounmask-keep-keywords=y --autounmask-use=y"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox -mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\ && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\ - && echo 'dev-vcs/git -webdev -gnome-keyring' | cat >> /etc/portage/package.use/git\ - && echo 'dev-util/buildbot-worker' | cat >> /etc/portage/package.accept_keywords/buildbot\ && echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\ - && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg rust-bin dev-vcs/git - #&& chown buildbot:buildbot /var/lib/buildbot_worker + && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin dev-vcs/git + +# We don't need the gentoo repo any longer +RUN rm -R /var/db/repos/gentoo + +# run on buildbot user +RUN chown buildbot:buildbot /var/lib/buildbot_worker +USER buildbot -#FIXME: run worker as buildbot (git fail) -#USER buildbot WORKDIR /var/lib/buildbot_worker -RUN wget https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac +# Get the needed buildbot-worker config +# Change umask so portage can read the ebuilds in the repos +RUN wget https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\ + && ( sed -i -e 's/umask = None/umask = 0o22/g' buildbot.tac 2>/dev/null || true ) ENTRYPOINT ["/usr/bin/buildbot-worker"] CMD ["start", "--nodaemon"]
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 5feaf00fc4770fbf3088244faa129996d1b08b8c Author: Magnus Granberg gentoo org> AuthorDate: Wed Oct 5 09:06:09 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Oct 5 09:06:09 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5feaf00f Fix the Log and Build dockerfiles Signed-off-by: Magnus Granberg gentoo.org> docker/GentooBuildbotWorker.Dockerfile | 14 docker/GentooBuildbotWorkerBuild.Dockerfile| 79 ++ ...ckerfile => GentooBuildbotWorkerLog.Dockerfile} | 0 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/docker/GentooBuildbotWorker.Dockerfile b/docker/GentooBuildbotWorker.Dockerfile deleted file mode 100644 index db5e398..000 --- a/docker/GentooBuildbotWorker.Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# This Dockerfile creates a gentoo buildbot worker container image -# from a gentoo project stage4 docker image. - -ARG PROJECTUUID - -FROM stage4-${PROJECTUUID} -RUN echo "Building Gentoo Buildbot worker Container image for ${PROJECTUUID}" \ - && ( sed -i -e 's/#rc_sys=""/rc_sys="docker"/g' etc/rc.conf 2>/dev/null || true ) \ - && echo 'UTC' > etc/timezone \ - && echo 'docker' >> /var/lib/buildbot_worker/info/host \ - && echo 'bb-worker-${PROJECTUUID}:latest' >> /var/lib/buildbot_worker/info/host -WORKDIR /var/lib/buildbot_worker -ENTRYPOINT ["/usr/bin/buildbot-worker"] -CMD ["start", "--nodaemon"] diff --git a/docker/GentooBuildbotWorkerBuild.Dockerfile b/docker/GentooBuildbotWorkerBuild.Dockerfile new file mode 100644 index 000..bd357dc --- /dev/null +++ b/docker/GentooBuildbotWorkerBuild.Dockerfile @@ -0,0 +1,79 @@ +# This Dockerfile creates a gentoo stage4 container image with buildbot worker installed. + +ARG TAG=latest + +FROM gentoo/portage:latest as portage + +FROM gentoo/stage3:$TAG + +ARG KEYWORDS +ARG PROJECTUUID +ARG BINHOSTURL +ARG ARCH +ARG TAG=latest +ARG NODE + +# copy the entire portage volume in +COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo + +# Setup portage for git install +RUN echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox -mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\ + && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf + +# Install git +RUN emerge -qv dev-vcs/git + +# Setup portage for the rest of the build +RUN echo -e "[binhost]\npriority = \nsync-uri = ${BINHOSTURL}/${PROJECTUUID}/packages/\n" | cat >> /etc/portage/binrepos.conf\ + && echo 'EMERGE_DEFAULT_OPTS="--buildpkg=y --rebuild-if-new-rev=y --rebuilt-binaries=y --usepkg=y --binpkg-respect-use=n --binpkg-changed-deps=y --nospinner --color=n --ask=n"' | cat >> /etc/portage/make.conf\ + && echo 'FEATURES="${FEATURES} parallel-fetch -merge-sync "' | cat >> /etc/portage/make.conf\ + && echo 'FEATURES="${FEATURES} getbinpkg"' | cat >> /etc/portage/make.conf\ + && echo 'COMMON_FLAGS="${COMMON_FLAGS} -fno-diagnostics-color -march=native"' | >> /etc/portage/make.conf\ + && echo 'USE="X elogind"' | >> /etc/portage/make.conf\ + && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >> /etc/portage/make.conf\ + && echo 'ACCEPT_LICENSE="*"'\ + && ( sed -i -e 's/#rc_sys=""/rc_sys="docker"/g' /etc/rc.conf 2>/dev/null || true ) \ + && echo 'UTC' > /etc/timezone + +# Setup LANG +RUN echo 'en_US.UTF-8 UTF-8' | cat >> /etc/locale.gen\ + && echo 'en_US ISO-8859-1' | cat >> /etc/locale.gen\ + && echo 'C.UTF8 UTF-8' | cat >> /etc/locale.gen\ + && echo 'LANG="en_US.utf8"' | cat >> /etc/locale.conf\ + && locale-gen + +# clone gentoo repo use github https for +# gitlab.gentoo.org do timeout on https +RUN rm -R /var/db/repos/gentoo\ + && git clone --depth=1 https://github.com/gentoo/gentoo.git /var/db/repos/gentoo + +# set accept_keywords +RUN echo 'ACCEPT_KEYWORDS="'"${KEYWORDS}"'"' | cat >> /etc/portage/make.conf + +# Setup world file +RUN echo 'app-text/ansifilter' | cat >> /var/lib/portage/world\ + && echo 'dev-util/pkgcheck' | cat >> /var/lib/portage/world\ + && echo 'dev-lang/rust-bin' | cat >> /var/lib/portage/world\ + && echo 'app-admin/eclean-kernel' | cat >> /var/lib/portage/world\ + && echo 'app-portage/gentoolkit' | cat >> /var/lib/portage/world\ + && echo 'app-editors/nano' | cat >> /var/lib/portage/world + +# Run emerge update and install buildbot-worker +RUN emerge -1qv perl\ + && perl-cleaner --all\ + && emerge -qv --update --deep --newuse @world\ + && emerge -qv gentoo-kernel-bin\ + && emerge -qv buildbot-worker + +# Rm the gentoo repo +RUN rm -R /var/db/repos/gentoo + +# Setup the buildbot-worker +WORKDIR /var/lib/buildbot_worker +RUN wget https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\ + && ( sed -i -e 's/umask = None/umask = 0o22/g' buildbot.tac 2>/dev/null || true ) \ + && mkdir info\ + && echo "${PROJECTUUID} ${ARCH} stage3:${TAG} ${NODE}" | cat > info/host +ENTRYPOINT ["/usr/bin/buildbot-worker"] +CMD ["start", "--nodaemon"] +#CMD ["/bin/bash"]
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 68ecdf96df4a062728672ecad2e275f7c17ee388 Author: Magnus Granberg gentoo org> AuthorDate: Thu Aug 18 14:39:06 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Thu Aug 18 14:46:53 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=68ecdf96 Fix line 14 on bb worker default docker file Thanx FurudeRika Signed-off-by: Magnus Granberg gentoo.org> docker/GentooBuildbotWorkerDefault.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/GentooBuildbotWorkerDefault.Dockerfile b/docker/GentooBuildbotWorkerDefault.Dockerfile index d1197c6..a7c7af1 100644 --- a/docker/GentooBuildbotWorkerDefault.Dockerfile +++ b/docker/GentooBuildbotWorkerDefault.Dockerfile @@ -11,7 +11,7 @@ COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo # emerge needed deps buildbot-worker, psycopg, git and sqlalchemy # get the needed buildbot-worker config RUN echo -e "[binhost]\npriority = \nsync-uri = https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n; | cat >> /etc/portage/binrepos.conf\ - && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y --autounmask-write --autounmask-continue --autounmask-keep-keywords=y --autounmask-use=y"' | cat >> /etc/portage/m> + && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y --autounmask-write --autounmask-continue --autounmask-keep-keywords=y --autounmask-use=y"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox -mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 51ab5319f6134dd57b74e4cc7a94caacd9941e83 Author: Magnus Granberg gentoo org> AuthorDate: Thu Jul 14 00:29:16 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Thu Jul 14 00:29:16 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=51ab5319 Add docker file for log docker image Signed-off-by: Magnus Granberg gentoo.org> docker/GentooBuildbotWorkerDefault.Dockerfile | 26 ++ 1 file changed, 26 insertions(+) diff --git a/docker/GentooBuildbotWorkerDefault.Dockerfile b/docker/GentooBuildbotWorkerDefault.Dockerfile new file mode 100644 index 000..7e1a63b --- /dev/null +++ b/docker/GentooBuildbotWorkerDefault.Dockerfile @@ -0,0 +1,26 @@ +# name the portage image +FROM gentoo/portage:latest as portage + +# based on stage3 image +FROM gentoo/stage3:latest + +# copy the entire portage volume in +COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo + +# Setup portage +# emerge needed deps buildbot-worker, psycopg and sqlalchemy +# get the needed buildbot-worker config +RUN echo -e "[binhost]\npriority = \nsync-uri = https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n; | cat >> /etc/portage/binrepos.conf\ + && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y --autounmask-write --autounmask-continue --autounmask-keep-keywords=y --autounmask-use=y"' | cat >> /etc/portage/make.conf\ + && echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox -mount-sandbox -sandbox"' | cat >> /etc/portage/make.conf\ + && echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"' | cat >> /etc/portage/make.conf\ + && echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\ + && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\ + && echo 'dev-util/buildbot-worker' | cat >> /etc/portage/package.accept_keywords/buildbot\ + && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg rust-bin\ + && wget https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\ + && cp buildbot.tac /var/lib/buildbot_worker/buildbot.tac + +WORKDIR /var/lib/buildbot_worker +ENTRYPOINT ["/usr/bin/buildbot-worker"] +CMD ["start", "--nodaemon"]
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: aa857a46bc666f780586ba62cebe7d6bb77fba2d Author: Magnus Granberg gentoo org> AuthorDate: Wed Jun 1 00:18:37 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Jun 1 00:18:37 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aa857a46 Add support for Building Docker Buildbot worker images Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/db/projects.py | 1 + buildbot_gentoo_ci/steps/nodes.py | 82 ++ docker/GentooBuildbotWorker.Dockerfile | 14 ++ 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py index dc91ce1..936cd9d 100644 --- a/buildbot_gentoo_ci/db/projects.py +++ b/buildbot_gentoo_ci/db/projects.py @@ -220,6 +220,7 @@ class ProjectsConnectorComponent(base.DBConnectorComponent): profile=row.profile, profile_repository_uuid=row.profile_repository_uuid, keyword_id=row.keyword_id, +image=row.image, status=row.status, auto=row.auto, enabled=row.enabled, diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index 37cf917..2a25562 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -42,7 +42,9 @@ class SetupPropertys(BuildStep): self.setProperty('project_data', project_data, 'project_data') #FIXME: set it in db node config self.workerbase = yield os.path.join('/', 'srv', 'gentoo', 'stage4') +self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid')) self.setProperty('workerbase', self.workerbase, 'workerbase') +self.setProperty('workerdest', self.workerdest, 'workerdest') self.setProperty('stage3', 'image', 'stage3') # we only support docker for now self.setProperty('type', 'docker', 'type') @@ -97,7 +99,7 @@ class SetupStage4Steps(BuildStep): aftersteps_list.append( steps.StringDownload(locale_conf_string + separator, workerdest="locale.gen", -workdir=workerdest_etc +workdir=self.getProperty("workerdest") + '/etc' )) yield log.addStdout('File: ' + 'locale.gen' + '\n') for line in locale_conf: @@ -105,7 +107,7 @@ class SetupStage4Steps(BuildStep): aftersteps_list.append( steps.StringDownload('LANG="en_US.utf8"' + separator, workerdest="locale.conf", -workdir=workerdest_etc +workdir=self.getProperty("workerdest") + '/etc' )) yield log.addStdout('Setting LANG to: ' + 'en_US.utf8' + '\n') aftersteps_list.append(SetSystemdNspawnConf()) @@ -117,16 +119,13 @@ class SetupStage4Steps(BuildStep): workdir=self.getProperty("workerbase") )) # update timezone -# install packages in world file config -command_list = ['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge'] -for package in package_list: -command_list.append(package) -aftersteps_list.append(steps.ShellCommand( -flunkOnFailure=True, -name='Install programs on the chroot', -command=command_list, -workdir=self.getProperty("workerbase") -)) +# add the world file +package_list_string = separator.join(package_list) +aftersteps_list.append( +steps.StringDownload(package_list_string + separator, +workerdest="var/lib/portage/world", +workdir=self.getProperty("workerdest") +)) # update container aftersteps_list.append(steps.ShellCommand( flunkOnFailure=True, @@ -141,25 +140,6 @@ class SetupStage4Steps(BuildStep): command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', 'buildbot-worker'], workdir=self.getProperty("workerbase") )) -#FIXME: move this to image build for chroot type part -if self.getProperty("type") == 'chroot': -# set hostname -aftersteps_list.append(steps.StringDownload( -self.getProperty("worker") + separator, -workerdest="hostname", -workdir=workerdest_etc -))