From 05fe89f718f58ba08fa376dc006a7fda144ab8fb Mon Sep 17 00:00:00 2001
From: Moritz Maxeiner <mm@ucw.sh>
Date: Mon, 15 Jul 2019 14:34:24 +0200
Subject: [PATCH] Update ci/cd
---
.gitlab-ci.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++++
.gitlab-ci.yml | 77 +++++++++++-------------------
2 files changed, 153 insertions(+), 49 deletions(-)
create mode 100755 .gitlab-ci.py
diff --git a/.gitlab-ci.py b/.gitlab-ci.py
new file mode 100755
index 0000000..e638c13
--- /dev/null
+++ b/.gitlab-ci.py
@@ -0,0 +1,125 @@
+#! /usr/bin/env python3
+
+import tarfile
+import shutil
+import ssl
+
+from os import environ as env
+from platform import system
+from subprocess import check_call, check_output
+from argparse import ArgumentParser
+from io import BytesIO
+from urllib.request import Request, urlopen
+from urllib.parse import quote, urlencode
+from zipfile import ZipFile
+from pathlib import Path
+
+
+def define(key: str, value: str):
+ return ['-D', f'{key}={value}']
+
+
+def define_env(name: str):
+ return define(name, env[name]) if name in env else []
+
+
+def fetch_artifacts(project, reference, job):
+ gitlab_host = 'https://git.imp.fu-berlin.de'
+ project = quote(project, safe="")
+ reference = quote(reference, safe="")
+ params = urlencode([("job", job)], doseq=True)
+ url = f'{gitlab_host}/api/v4/projects/{project}/jobs/artifacts/{reference}/download?{params}'
+ headers = {'JOB-TOKEN': env['CI_JOB_TOKEN']}
+ return ZipFile(
+ BytesIO(
+ urlopen(Request(url, headers=headers),
+ context=ssl._create_unverified_context()).read()))
+
+
+def extract_cmake_package(artifacts, name):
+ for filename in artifacts.namelist():
+ if Path(filename).match(f'{name}-*.tar.xz'):
+ with tarfile.open(fileobj=BytesIO(artifacts.read(filename))) as f:
+ f.extractall('vendor')
+ shutil.move(next(Path('vendor').glob(f'{name}-*/')),
+ f'vendor/{name}')
+
+
+if system() == 'Windows':
+
+ def setup_msvc():
+ msvc_path = 'C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/Tools'
+ lines = check_output([
+ 'cmd', '/c', 'VsDevCmd.bat', '-arch=amd64',
+ f'-vcvars_ver={env["VCVARS_VER"]}', '&', 'set'
+ ],
+ cwd=msvc_path).decode('utf-8').splitlines()
+ for line in lines:
+ split = line.split('=')
+ if len(split) != 2:
+ continue
+ key, value = split
+ if key in env and env[key] == value:
+ continue
+ env[key] = value
+
+
+def prepare(args):
+ for name, project, reference, job in args.dependencies:
+ with fetch_artifacts(project, reference, job) as artifacts:
+ extract_cmake_package(artifacts, name)
+
+
+def build(args):
+ if system() == 'Windows':
+ setup_msvc()
+
+ command = ['cmake']
+ command += ['-S', '.']
+ command += ['-B', 'build']
+ command += ['-G', 'Ninja']
+ command += define('CMAKE_PREFIX_PATH', 'vendor')
+ command += define_env('CMAKE_BUILD_TYPE')
+ command += define('CMAKE_SUPPRESS_REGENERATION', 'ON')
+ command += define('CMAKE_SKIP_PACKAGE_ALL_DEPENDENCY', 'ON')
+
+ if system() == 'Windows':
+ command += define(
+ 'CMAKE_TOOLCHAIN_FILE',
+ env['VCPKG_DIR'] + '/scripts/buildsystems/vcpkg.cmake')
+ command += define('VCPKG_TARGET_TRIPLET', env['VCPKG_TRIPLET'])
+ command += define('PACKAGE_WINDOWS_MSI', 'ON')
+ elif system() == 'Linux':
+ command += define('PACKAGE_TXZ', 'ON')
+ check_call(command)
+
+ command = ['ninja', '-C', 'build']
+ check_call(command)
+
+
+def package(args):
+ command = ['ninja', '-C', 'build', 'package']
+ check_call(command)
+
+
+if __name__ == '__main__':
+ parser = ArgumentParser()
+ subparsers = parser.add_subparsers()
+
+ prepare_parser = subparsers.add_parser('prepare')
+ prepare_parser.set_defaults(task=prepare)
+ prepare_parser.add_argument('--dependency',
+ dest='dependencies',
+ nargs=4,
+ action='append',
+ metavar=('PACKAGE', 'PROJECT', 'REFERENCE',
+ 'JOB'))
+
+ build_parser = subparsers.add_parser('build')
+ build_parser.set_defaults(task=build)
+
+ package_parser = subparsers.add_parser('package')
+ package_parser.set_defaults(task=package)
+
+ args = parser.parse_args()
+ args.task(args)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e42624e..5edeeb9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,13 +3,14 @@ stages:
- package
- deploy
+
.ubuntu-18.04:
tags: [ linux, docker ]
image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-ubuntu18.04
-.windows:
- tags: [ windows, docker ]
- image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows
+.windows-1809:
+ tags: [ windows-1809, docker ]
+ image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows1809
.gcc8: &gcc8
@@ -17,60 +18,44 @@ stages:
CXX: g++-8
.msvc15.9: &msvc15_9
- VSDevEnv -arch=amd64 -vcvars_ver="14.16"
-
-.debug: &debug
- CMAKE_BUILD_TYPE: Debug
+ VCVARS_VER: '14.16'
.release: &release
CMAKE_BUILD_TYPE: Release
+.debug: &debug
+ CMAKE_BUILD_TYPE: Debug
-.build ubuntu-18.04:
- extends: .ubuntu-18.04
- stage: build
- artifacts:
- paths:
- - build
- expire_in: 1 day
- script:
- - cmake -Bbuild -H. -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DPACKAGE_TXZ=ON -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON
- - ninja -C build
-.build windows:
- extends: .windows
+.build: &build
stage: build
artifacts:
paths:
+ - vendor
- build
expire_in: 1 day
- script:
- - cmake -Bbuild "-H." -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DPACKAGE_WINDOWS_MSI=ON -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_DIR/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET="$env:VCPKG_TRIPLET" -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON -DCPACK_PACKAGE_DIRECTORY="${env:PACKAGE_DIR}"
- - ninja -C build
+ script: ./.gitlab-ci.py build
build ubuntu-18.04:
- extends: .build ubuntu-18.04
+ extends: .ubuntu-18.04
+ <<: *build
variables:
<<: [ *gcc8, *release ]
before_script:
- - . /etc/profile.d/robofish.sh
- - gitlab-fetch-artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20ubuntu-18.04
- - cmake-integrate-package biotracker-interfaces
- - gitlab-fetch-artifacts bioroboticslab%2Fbiotracker%2Futility master package%20ubuntu-18.04
- - cmake-integrate-package biotracker-utility
-
-build windows:
- extends: .build windows
+ - ./.gitlab-ci.py prepare
+ --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package ubuntu-18.04'
+ --dependency biotracker-utility bioroboticslab/biotracker/utility master 'package ubuntu-18.04'
+
+build windows-1809:
+ extends: .windows-1809
+ <<: *build
variables:
- <<: [ *release ]
+ <<: [ *msvc15_9, *release ]
before_script:
- - . $Profile
- - *msvc15_9
- - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20windows
- - CMake-Integrate-Package biotracker-interfaces
- - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Futility master package%20windows
- - CMake-Integrate-Package biotracker-utility
- - $env:PACKAGE_DIR = ("${env:TMP}/pkg" -Replace "\\","/")
+ - ./.gitlab-ci.py prepare
+ --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package windows-1809'
+ --dependency biotracker-utility bioroboticslab/biotracker/utility master 'package windows-1809'
+
.package: &package
stage: package
@@ -79,8 +64,7 @@ build windows:
- build/*.tar.xz
- build/*.msi
expire_in: 1 week
- script:
- - ninja -C build package
+ script: ./.gitlab-ci.py package
package ubuntu-18.04:
extends: .ubuntu-18.04
@@ -88,13 +72,8 @@ package ubuntu-18.04:
- build ubuntu-18.04
<<: *package
-package windows:
- extends: .windows
+package windows-1809:
+ extends: .windows-1809
dependencies:
- - build windows
+ - build windows-1809
<<: *package
- before_script:
- - $env:PACKAGE_DIR = ("${env:TMP}/pkg" -Replace "\\","/")
- script:
- - ninja -C build package
- - Copy-Item "$env:PACKAGE_DIR/*" build -Recurse
--
GitLab