BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/291525
Change subject: Make the builder script less simple
......................................................................
Make the builder script less simple
* Generate Dockerfile from Dockerfile.template
* Do not build siblings
* Add unit tests and flake8 linting with tox
* Allow alternate DOCKER_BINARY via environment variable
* Allow python3 from PATH for osx and virtualenv runtimes
Change-Id: Ifbf155de741dda25636989269bc66c332bc62f6e
---
A .gitignore
A README.md
A base/.dockerignore
R base/Dockerfile.template
M build.py
A java/base/.dockerignore
D java/base/Dockerfile
A java/base/Dockerfile.template
A java/web/.dockerignore
D java/web/Dockerfile
A java/web/Dockerfile.template
A php/base/.dockerignore
R php/base/Dockerfile.template
A php/web/.dockerignore
R php/web/Dockerfile.template
A tests/build_tests.py
A tox.ini
17 files changed, 164 insertions(+), 26 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/operations/docker-images/toollabs-images
refs/changes/25/291525/1
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..76b725c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+*.py[cod]
+/.tox
+__pycache__/
+Dockerfile
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..3c776ba
--- /dev/null
+++ b/README.md
@@ -0,0 +1,6 @@
+Tool Labs Images
+================
+
+Docker image configuration and management for Wikimedia Tool Labs
+
+TODO: describe how this works
diff --git a/base/.dockerignore b/base/.dockerignore
new file mode 100644
index 0000000..689cf97
--- /dev/null
+++ b/base/.dockerignore
@@ -0,0 +1 @@
+Dockerfile.template
diff --git a/base/Dockerfile b/base/Dockerfile.template
similarity index 91%
rename from base/Dockerfile
rename to base/Dockerfile.template
index e6bcc66..2a1fcb1 100644
--- a/base/Dockerfile
+++ b/base/Dockerfile.template
@@ -1,5 +1,5 @@
# Base toollabs user image
-# Name: docker-registry.tools.wmflabs.org/jessie-toollabs
+# Name: docker-registry.tools.wmflabs.org/toollabs-base
# Source image: Debian Jessie
# Sets up:
# Extra Apt repo: Tools repo
diff --git a/build.py b/build.py
index 62c2f15..eb41b6f 100755
--- a/build.py
+++ b/build.py
@@ -1,11 +1,14 @@
-#!/usr/bin/python3
-import subprocess
-import os
+#!/usr/bin/env python3
+"""
+Build and publish Docker images.
+"""
import argparse
+import os
+import subprocess
# The docker binary to use for executing commands
-DOCKER_BINARY = '/usr/bin/docker'
+DOCKER_BINARY = os.environ.get('DOCKER_BINARY', '/usr/bin/docker')
# Base path of where the docker images are organized
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
@@ -22,8 +25,8 @@
],
'java/base': [
'java/web',
- ]
- }
+ ],
+ },
}
@@ -35,7 +38,19 @@
)
+def make_dockerfile(name, registry, image_prefix):
+ image_dir = os.path.join(BASE_PATH, name)
+ template_file = os.path.join(image_dir, 'Dockerfile.template')
+ out_file = os.path.join(image_dir, 'Dockerfile')
+ kwargs = {'registry': registry, 'image_prefix': image_prefix}
+ with open(template_file, 'rt') as f_in:
+ with open(out_file, 'wt') as f_out:
+ for line in f_in:
+ f_out.write(expand_template(line, kwargs))
+
+
def build_image(name, registry, image_prefix):
+ make_dockerfile(name, registry, image_prefix)
subprocess.check_call([
DOCKER_BINARY,
'build',
@@ -54,19 +69,23 @@
def lineage_of(name):
- def children_of(val):
- if type(val) == dict:
- children = list(val.keys())
- for k, v in val.items():
+ def children_of(node):
+ if type(node) == dict:
+ children = list(node.keys())
+ for k, v in node.items():
children += children_of(v)
return children
- return val
+ return node
- def ancestors_of(val, cur_lineage):
- if name in val:
- return cur_lineage + children_of(val)
- if type(val) == dict:
- for k, v in val.items():
+ def ancestors_of(node, cur_lineage):
+ if name in node:
+ cur_lineage.append(name)
+ if type(node) == dict:
+ cur_lineage.extend(children_of(node[name]))
+ return cur_lineage
+
+ if type(node) == dict:
+ for k, v in node.items():
ret = ancestors_of(v, cur_lineage + [k])
if ret:
return ret
@@ -75,6 +94,10 @@
return ancestors_of(IMAGES, [])
+def expand_template(template, params):
+ return template.format(**params)
+
+
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument(
diff --git a/java/base/.dockerignore b/java/base/.dockerignore
new file mode 100644
index 0000000..689cf97
--- /dev/null
+++ b/java/base/.dockerignore
@@ -0,0 +1 @@
+Dockerfile.template
diff --git a/java/base/Dockerfile b/java/base/Dockerfile
deleted file mode 100644
index e7f4f48..0000000
--- a/java/base/Dockerfile
+++ /dev/null
@@ -1,4 +0,0 @@
-FROM docker-registry.tools.wmflabs.org/toollabs-jessie
-
-RUN apt-get install --yes -t jessie-backports \
- openjdk-8-jdk
diff --git a/java/base/Dockerfile.template b/java/base/Dockerfile.template
new file mode 100644
index 0000000..6a93891
--- /dev/null
+++ b/java/base/Dockerfile.template
@@ -0,0 +1,6 @@
+# Toollabs Java runtime image
+# Name: docker-registry.tools.wmflabs.org/toollabs-java
+FROM {registry}/{image_prefix}-base
+
+RUN apt-get install --yes -t jessie-backports \
+ openjdk-8-jdk
diff --git a/java/web/.dockerignore b/java/web/.dockerignore
new file mode 100644
index 0000000..689cf97
--- /dev/null
+++ b/java/web/.dockerignore
@@ -0,0 +1 @@
+Dockerfile.template
diff --git a/java/web/Dockerfile b/java/web/Dockerfile
deleted file mode 100644
index 9786300..0000000
--- a/java/web/Dockerfile
+++ /dev/null
@@ -1,3 +0,0 @@
-FROM docker-registry.tools.wmflabs.org/toollabs-java
-
-RUN apt-get install --yes toollabs-webservice
diff --git a/java/web/Dockerfile.template b/java/web/Dockerfile.template
new file mode 100644
index 0000000..b7c32be
--- /dev/null
+++ b/java/web/Dockerfile.template
@@ -0,0 +1,5 @@
+# Toollabs Java webserver image
+# Name: docker-registry.tools.wmflabs.org/toollabs-java-web
+FROM {registry}/{image_prefix}-java
+
+RUN apt-get install --yes toollabs-webservice
diff --git a/php/base/.dockerignore b/php/base/.dockerignore
new file mode 100644
index 0000000..689cf97
--- /dev/null
+++ b/php/base/.dockerignore
@@ -0,0 +1 @@
+Dockerfile.template
diff --git a/php/base/Dockerfile b/php/base/Dockerfile.template
similarity index 65%
rename from php/base/Dockerfile
rename to php/base/Dockerfile.template
index 5855d56..6988d5d 100644
--- a/php/base/Dockerfile
+++ b/php/base/Dockerfile.template
@@ -1,4 +1,6 @@
-FROM docker-registry.tools.wmflabs.org/toollabs-jessie
+# Toollabs PHP runtime image
+# Name: docker-registry.tools.wmflabs.org/toollabs-php
+FROM {registry}/{image_prefix}-base
RUN apt-get install --yes \
php5-cli \
diff --git a/php/web/.dockerignore b/php/web/.dockerignore
new file mode 100644
index 0000000..689cf97
--- /dev/null
+++ b/php/web/.dockerignore
@@ -0,0 +1 @@
+Dockerfile.template
diff --git a/php/web/Dockerfile b/php/web/Dockerfile.template
similarity index 68%
rename from php/web/Dockerfile
rename to php/web/Dockerfile.template
index fcf1ba1..4aa3145 100644
--- a/php/web/Dockerfile
+++ b/php/web/Dockerfile.template
@@ -1,4 +1,6 @@
-FROM docker-registry.tools.wmflabs.org/toollabs-php
+# Toollabs PHP webserver image
+# Name: docker-registry.tools.wmflabs.org/toollabs-php-web
+FROM {registry}/{image_prefix}-php
RUN apt-get install --yes \
lighttpd \
diff --git a/tests/build_tests.py b/tests/build_tests.py
new file mode 100644
index 0000000..e4c4700
--- /dev/null
+++ b/tests/build_tests.py
@@ -0,0 +1,68 @@
+import build
+from nose.tools import assert_equals
+
+
+def assert_same(result, expect):
+ """Check arrays for order-free equality."""
+ assert_equals(sorted(result), sorted(expect))
+
+
+def test_lineage_of():
+ build.IMAGES = {
+ 'base': {
+ 'A': ['A1', 'A2'],
+ 'B': ['B1', 'B2'],
+ 'C': {
+ 'C1': {'C1a': ['C1a1']},
+ 'C2': ['C2a'],
+ }
+ }
+ }
+
+ assert_equals(build.lineage_of('not_found'), None)
+ assert_same(
+ build.lineage_of('base'),
+ [
+ 'base',
+ 'A', 'A1', 'A2',
+ 'B', 'B1', 'B2',
+ 'C', 'C1', 'C1a', 'C1a1', 'C2', 'C2a',
+ ]
+ )
+ assert_same(
+ build.lineage_of('B1'),
+ ['base', 'B', 'B1']
+ )
+ assert_same(
+ build.lineage_of('C1'),
+ ['C1a1', 'C1a', 'C1', 'C', 'base']
+ )
+ assert_same(
+ build.lineage_of('C1a1'),
+ ['C1a1', 'C1a', 'C1', 'C', 'base']
+ )
+
+
+def test_make_docker_tag():
+ tests = {
+ 'registry/prefix-name': ['name', 'registry', 'prefix'],
+ 'test/prefix-foo-bar-baz': ['foo/bar/baz', 'test', 'prefix'],
+ }
+ for expect, args in tests.items():
+ yield check_make_docker_tag, args, expect
+
+
+def check_make_docker_tag(given, expect):
+ assert_equals(build.make_docker_tag(*given), expect)
+
+
+def test_expand_template():
+ template = "foo:{foo}\nbar:{bar}"
+ assert_equals(
+ build.expand_template(template, {'foo': 1, 'bar': 2}),
+ "foo:1\nbar:2"
+ )
+ assert_equals(
+ build.expand_template(template, {'foo': 1, 'bar': 2, 'baz': 3}),
+ "foo:1\nbar:2"
+ )
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..5ecb708
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,24 @@
+[tox]
+envlist = py3, flake8
+minversion = 1.6
+skipsdist = True
+
+[testenv]
+basepython = python3
+commands = nosetests tests
+deps = nose
+
+[testenv:flake8]
+basepython = python3
+commands = flake8
+deps = flake8
+
+[flake8]
+max-line-length = 100
+# E121, E123, E126, E133, E226, E241, E242, E704 and W503 are default ignores
+# E124: closing bracket does not match visual indentation
+ignore = E121,E123,E126,E133,E226,E241,E242,E704,W503,E124
+
+[nosetests]
+verbosity=3
+with-doctest=1
--
To view, visit https://gerrit.wikimedia.org/r/291525
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifbf155de741dda25636989269bc66c332bc62f6e
Gerrit-PatchSet: 1
Gerrit-Project: operations/docker-images/toollabs-images
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
