diff options
author | Ulysses Souza <ulysses.souza@docker.com> | 2019-04-12 18:46:06 +0200 |
---|---|---|
committer | Ulysses Souza <ulysses.souza@docker.com> | 2019-04-17 16:08:33 +0200 |
commit | 2b24eb693c62ce8fd5f274f0fcb2837132a6a0b8 (patch) | |
tree | ead40d5ed0857db14fb6c0cd253287b5a54fcdd4 | |
parent | c217bab7f6123de80dbd55c99c2254666d766fb3 (diff) |
Refactor release and build scripts
- Make use of the same Dockerfile when producing
an image for testing and for deploying to
DockerHub
Signed-off-by: Ulysses Souza <ulysses.souza@docker.com>
-rw-r--r-- | Jenkinsfile | 8 | ||||
-rwxr-xr-x | script/build/image | 11 | ||||
-rwxr-xr-x | script/build/linux | 15 | ||||
-rwxr-xr-x | script/build/linux-entrypoint | 19 | ||||
-rwxr-xr-x | script/build/osx | 5 | ||||
-rwxr-xr-x | script/build/test-image | 8 | ||||
-rwxr-xr-x | script/build/write-git-sha | 2 | ||||
-rwxr-xr-x | script/release/release.py | 13 | ||||
-rw-r--r-- | script/release/release/images.py | 101 | ||||
-rw-r--r-- | script/release/release/repository.py | 1 | ||||
-rwxr-xr-x | script/run/run.sh | 2 | ||||
-rwxr-xr-x | script/test/all | 3 | ||||
-rwxr-xr-x | script/test/default | 6 |
13 files changed, 126 insertions, 68 deletions
diff --git a/Jenkinsfile b/Jenkinsfile index 51fecf99..4de276ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,13 @@ def buildImage = { String baseImage -> try { image.pull() } catch (Exception exc) { - sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ." + sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\ + docker build -t ${imageName} \\ + --target build \\ + --build-arg BUILD_PLATFORM="${baseImage}" \\ + --build-arg GIT_COMMIT="${GIT_COMMIT}" \\ + .\\ + """ sh "docker push ${imageName}" echo "${imageName}" return imageName diff --git a/script/build/image b/script/build/image index a3198c99..fb3f856e 100755 --- a/script/build/image +++ b/script/build/image @@ -7,11 +7,14 @@ if [ -z "$1" ]; then exit 1 fi -TAG=$1 +TAG="$1" VERSION="$(python setup.py --version)" -./script/build/write-git-sha +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA python setup.py sdist bdist_wheel -./script/build/linux -docker build -t docker/compose:$TAG -f Dockerfile.run . + +docker build \ + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" \ + -t "${TAG}" . diff --git a/script/build/linux b/script/build/linux index 8de7218d..28065da0 100755 --- a/script/build/linux +++ b/script/build/linux @@ -4,15 +4,14 @@ set -ex ./script/clean -TMP_CONTAINER="tmpcontainer" -TAG="docker/compose:tmp-glibc-linux-binary" -DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +TAG="docker/compose:tmp-glibc-linux-binary-${DOCKER_COMPOSE_GITSHA}" docker build -t "${TAG}" . \ --build-arg BUILD_PLATFORM=debian \ - --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} -docker create --name ${TMP_CONTAINER} ${TAG} + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" +TMP_CONTAINER=$(docker create "${TAG}") mkdir -p dist -docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 -docker container rm -f ${TMP_CONTAINER} -docker image rm -f ${TAG} +docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 +docker container rm -f "${TMP_CONTAINER}" +docker image rm -f "${TAG}" diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index 1556bbf2..1c5438d8 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -3,16 +3,19 @@ set -ex CODE_PATH=/code -VENV=${CODE_PATH}/.tox/py37 +VENV="${CODE_PATH}"/.tox/py37 -cd ${CODE_PATH} +cd "${CODE_PATH}" mkdir -p dist chmod 777 dist -${VENV}/bin/pip3 install -q -r requirements-build.txt +"${VENV}"/bin/pip3 install -q -r requirements-build.txt # TODO(ulyssessouza) To check if really needed -./script/build/write-git-sha +if [ -z "${DOCKER_COMPOSE_GITSHA}" ]; then + DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +fi +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA export PATH="${CODE_PATH}/pyinstaller:${PATH}" @@ -21,15 +24,15 @@ if [ ! -z "${BUILD_BOOTLOADER}" ]; then git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller cd /tmp/pyinstaller/bootloader git checkout v3.4 - ${VENV}/bin/python3 ./waf configure --no-lsb all - ${VENV}/bin/pip3 install .. - cd ${CODE_PATH} + "${VENV}"/bin/python3 ./waf configure --no-lsb all + "${VENV}"/bin/pip3 install .. + cd "${CODE_PATH}" rm -Rf /tmp/pyinstaller else echo "NOT compiling bootloader!!!" fi -${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec +"${VENV}"/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec ls -la dist/ ldd dist/docker-compose mv dist/docker-compose /usr/local/bin diff --git a/script/build/osx b/script/build/osx index c62b2702..52991458 100755 --- a/script/build/osx +++ b/script/build/osx @@ -5,11 +5,12 @@ TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)" rm -rf venv -virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv +virtualenv -p "${TOOLCHAIN_PATH}"/bin/python3 venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . -./script/build/write-git-sha +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/build/test-image b/script/build/test-image index 9d880c27..4964a5f9 100755 --- a/script/build/test-image +++ b/script/build/test-image @@ -10,9 +10,9 @@ fi TAG="$1" IMAGE="docker/compose-tests" -DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" docker build -t "${IMAGE}:${TAG}" . \ --target build \ - --build-arg BUILD_PLATFORM=debian \ - --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} -docker tag ${IMAGE}:${TAG} ${IMAGE}:latest + --build-arg BUILD_PLATFORM="debian" \ + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" +docker tag "${IMAGE}":"${TAG}" "${IMAGE}":latest diff --git a/script/build/write-git-sha b/script/build/write-git-sha index be87f505..cac4b6fd 100755 --- a/script/build/write-git-sha +++ b/script/build/write-git-sha @@ -9,4 +9,4 @@ if [[ "${?}" != "0" ]]; then echo "Couldn't get revision of the git repository. Setting to 'unknown' instead" DOCKER_COMPOSE_GITSHA="unknown" fi -echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA +echo "${DOCKER_COMPOSE_GITSHA}" diff --git a/script/release/release.py b/script/release/release.py index 9db1a49d..9fdd92da 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -204,7 +204,7 @@ def resume(args): delete_assets(gh_release) upload_assets(gh_release, files) img_manager = ImageManager(args.release) - img_manager.build_images(repository, files) + img_manager.build_images(repository) except ScriptError as e: print(e) return 1 @@ -244,7 +244,7 @@ def start(args): gh_release = create_release_draft(repository, args.release, pr_data, files) upload_assets(gh_release, files) img_manager = ImageManager(args.release) - img_manager.build_images(repository, files) + img_manager.build_images(repository) except ScriptError as e: print(e) return 1 @@ -258,7 +258,8 @@ def finalize(args): try: check_pypirc() repository = Repository(REPO_ROOT, args.repo) - img_manager = ImageManager(args.release) + tag_as_latest = _check_if_tag_latest(args.release) + img_manager = ImageManager(args.release, tag_as_latest) pr_data = repository.find_release_pr(args.release) if not pr_data: raise ScriptError('No PR found for {}'.format(args.release)) @@ -314,6 +315,12 @@ EPILOG = '''Example uses: ''' +# Checks if this version respects the GA version format ('x.y.z') and not an RC +def _check_if_tag_latest(version): + ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2 + return ga_version and yesno('Should this release be tagged as \"latest\"? Y/n', default=True) + + def main(): if 'GITHUB_TOKEN' not in os.environ: print('GITHUB_TOKEN environment variable must be set') diff --git a/script/release/release/images.py b/script/release/release/images.py index df6eeda4..796a4d82 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -5,18 +5,29 @@ from __future__ import unicode_literals import base64 import json import os -import shutil import docker +from enum import Enum +from .const import NAME from .const import REPO_ROOT from .utils import ScriptError +class Platform(Enum): + ALPINE = 'alpine' + DEBIAN = 'debian' + + def __str__(self): + return self.value + + class ImageManager(object): - def __init__(self, version): + def __init__(self, version, latest=False): + self.built_tags = [] self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version + self.latest = latest if 'HUB_CREDENTIALS' in os.environ: print('HUB_CREDENTIALS found in environment, issuing login') credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) @@ -24,16 +35,31 @@ class ImageManager(object): username=credentials['Username'], password=credentials['Password'] ) - def build_images(self, repository, files): - print("Building release images...") - repository.write_git_sha() - distdir = os.path.join(REPO_ROOT, 'dist') - os.makedirs(distdir, exist_ok=True) - shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) - os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) - print('Building docker/compose image') + def _tag(self, image, existing_tag, new_tag): + existing_repo_tag = '{image}:{tag}'.format(image=image, tag=existing_tag) + new_repo_tag = '{image}:{tag}'.format(image=image, tag=new_tag) + self.docker_client.tag(existing_repo_tag, new_repo_tag) + self.built_tags.append(new_repo_tag) + + def build_runtime_image(self, repository, platform): + git_sha = repository.write_git_sha() + compose_image_base_name = NAME + print('Building {image} image ({platform} based)'.format( + image=compose_image_base_name, + platform=platform + )) + full_version = '{version}-{platform}'.format(version=self.version, platform=platform) + build_tag = '{image_base_image}:{full_version}'.format( + image_base_image=compose_image_base_name, + full_version=full_version + ) logstream = self.docker_client.build( - REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', + REPO_ROOT, + tag=build_tag, + buildargs={ + 'BUILD_PLATFORM': platform.value, + 'GIT_COMMIT': git_sha, + }, decode=True ) for chunk in logstream: @@ -42,9 +68,32 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - print('Building test image (for UCP e2e)') + self.built_tags.append(build_tag) + if platform == Platform.ALPINE: + self._tag(compose_image_base_name, full_version, self.version) + if self.latest: + self._tag(compose_image_base_name, full_version, platform) + if platform == Platform.ALPINE: + self._tag(compose_image_base_name, full_version, 'latest') + + # Used for producing a test image for UCP + def build_ucp_test_image(self, repository): + print('Building test image (debian based for UCP e2e)') + git_sha = repository.write_git_sha() + compose_tests_image_base_name = NAME + '-tests' + ucp_test_image_tag = '{image}:{tag}'.format( + image=compose_tests_image_base_name, + tag=self.version + ) logstream = self.docker_client.build( - REPO_ROOT, tag='docker-compose-tests:tmp', decode=True + REPO_ROOT, + tag=ucp_test_image_tag, + target='build', + buildargs={ + 'BUILD_PLATFORM': Platform.DEBIAN.value, + 'GIT_COMMIT': git_sha, + }, + decode=True ) for chunk in logstream: if 'error' in chunk: @@ -52,26 +101,16 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - container = self.docker_client.create_container( - 'docker-compose-tests:tmp', entrypoint='tox' - ) - self.docker_client.commit(container, 'docker/compose-tests', 'latest') - self.docker_client.tag( - 'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version) - ) - self.docker_client.remove_container(container, force=True) - self.docker_client.remove_image('docker-compose-tests:tmp', force=True) + self.built_tags.append(ucp_test_image_tag) + self._tag(compose_tests_image_base_name, self.version, 'latest') - @property - def image_names(self): - return [ - 'docker/compose-tests:latest', - 'docker/compose-tests:{}'.format(self.version), - 'docker/compose:{}'.format(self.version) - ] + def build_images(self, repository): + self.build_runtime_image(repository, Platform.ALPINE) + self.build_runtime_image(repository, Platform.DEBIAN) + self.build_ucp_test_image(repository) def check_images(self): - for name in self.image_names: + for name in self.built_tags: try: self.docker_client.inspect_image(name) except docker.errors.ImageNotFound: @@ -80,7 +119,7 @@ class ImageManager(object): return True def push_images(self): - for name in self.image_names: + for name in self.built_tags: print('Pushing {} to Docker Hub'.format(name)) logstream = self.docker_client.push(name, stream=True, decode=True) for chunk in logstream: diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 9a5d432c..0dc724f8 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -175,6 +175,7 @@ class Repository(object): def write_git_sha(self): with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f: f.write(self.git_repo.head.commit.hexsha[:7]) + return self.git_repo.head.commit.hexsha[:7] def cherry_pick_prs(self, release_branch, ids): if not ids: diff --git a/script/run/run.sh b/script/run/run.sh index a8690cad..f3456720 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -48,7 +48,7 @@ fi # Only allocate tty if we detect one if [ -t 0 -a -t 1 ]; then - DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" + DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" fi # Always set -i to support piped and terminal input in run/exec diff --git a/script/test/all b/script/test/all index 5c911bba..f929a57e 100755 --- a/script/test/all +++ b/script/test/all @@ -8,8 +8,7 @@ set -e docker run --rm \ --tty \ ${GIT_VOLUME} \ - --entrypoint="tox" \ - "$TAG" -e pre-commit + "$TAG" tox -e pre-commit get_versions="docker run --rm --entrypoint=/code/.tox/py27/bin/python diff --git a/script/test/default b/script/test/default index d24b41b0..4d973d1d 100755 --- a/script/test/default +++ b/script/test/default @@ -5,16 +5,16 @@ set -ex TAG="docker-compose:alpine-$(git rev-parse --short HEAD)" -# By default use the Dockerfile.alpine, but can be overridden to use an alternative file +# By default use the Dockerfile, but can be overridden to use an alternative file # e.g DOCKERFILE=Dockerfile.armhf script/test/default -DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}" +DOCKERFILE="${DOCKERFILE:-Dockerfile}" DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}" rm -rf coverage-html # Create the host directory so it's owned by $USER mkdir -p coverage-html -docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . +docker build -f "${DOCKERFILE}" -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" . script/test/all |