From 4a9d8bb80a9dbebf0c2ab42915e853ae22fe43f0 Mon Sep 17 00:00:00 2001
From: Moritz Maxeiner <mm@ucw.sh>
Date: Mon, 15 Jul 2019 14:29:59 +0200
Subject: [PATCH] Update ci/cd
---
.gitlab-ci.py | 161 +++++++++++++++++++++++++++++++++++++++++++++++++
.gitlab-ci.yml | 91 ++++++++++++----------------
2 files changed, 198 insertions(+), 54 deletions(-)
create mode 100755 .gitlab-ci.py
diff --git a/.gitlab-ci.py b/.gitlab-ci.py
new file mode 100755
index 0000000..a9f13ad
--- /dev/null
+++ b/.gitlab-ci.py
@@ -0,0 +1,161 @@
+#! /usr/bin/env python3
+
+import tarfile
+import shutil
+import ssl
+
+from os import environ as env, symlink
+from platform import system
+from subprocess import check_call, check_output, DEVNULL
+from argparse import ArgumentParser
+from io import BytesIO
+from urllib.request import Request, urlopen
+from urllib.parse import quote, urlencode
+from zipfile import ZipFile
+from pathlib import Path
+
+
+def define(key: str, value: str):
+ return ['-D', f'{key}={value}']
+
+
+def define_env(name: str):
+ return define(name, env[name]) if name in env else []
+
+
+def fetch_artifacts(project, reference, job):
+ gitlab_host = 'https://git.imp.fu-berlin.de'
+ project = quote(project, safe="")
+ reference = quote(reference, safe="")
+ params = urlencode([("job", job)], doseq=True)
+ url = f'{gitlab_host}/api/v4/projects/{project}/jobs/artifacts/{reference}/download?{params}'
+ headers = {'JOB-TOKEN': env['CI_JOB_TOKEN']}
+ return ZipFile(
+ BytesIO(
+ urlopen(Request(url,
+ headers=headers),
+ context=ssl._create_unverified_context()).read()
+ )
+ )
+
+
+def extract_cmake_package(artifacts, name):
+ for filename in artifacts.namelist():
+ if Path(filename).match(f'{name}-*.tar.xz'):
+ with tarfile.open(fileobj=BytesIO(artifacts.read(filename))) as f:
+ f.extractall('vendor')
+ shutil.move(next(Path('vendor').glob(f'{name}-*/')), f'vendor/{name}')
+
+
+def enable_cmake_package_discovery(name):
+ if system() == 'Windows':
+ check_call(
+ [
+ 'powershell',
+ '&',
+ 'cmd.exe',
+ '/c',
+ 'mklink',
+ '/J',
+ f'"C:/Program Files/{name}"',
+ f'"$(resolve-path vendor/{name})"'
+ ],
+ stdout=DEVNULL
+ )
+ elif system() == 'Linux':
+ symlink(Path(f'vendor/{name}').resolve(), f'/usr/local/{name}')
+ else:
+ assert False
+
+
+def integrate_cmake_package(artifacts, name):
+ extract_cmake_package(artifacts, name)
+ enable_cmake_package_discovery(name)
+
+
+if system() == 'Windows':
+
+ def setup_msvc():
+ msvc_path = 'C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/Tools'
+ lines = check_output(
+ [
+ 'cmd',
+ '/c',
+ 'VsDevCmd.bat',
+ '-arch=amd64',
+ f'-vcvars_ver={env["VCVARS_VER"]}',
+ '&',
+ 'set'
+ ],
+ cwd=msvc_path
+ ).decode('utf-8').splitlines()
+ for line in lines:
+ split = line.split('=')
+ if len(split) != 2:
+ continue
+ key, value = split
+ if key in env and env[key] == value:
+ continue
+ env[key] = value
+
+
+def prepare(args):
+ for name, project, reference, job in args.dependencies:
+ with fetch_artifacts(project, reference, job) as artifacts:
+ integrate_cmake_package(artifacts, name)
+
+
+def build(args):
+ if system() == 'Windows':
+ setup_msvc()
+
+ command = ['cmake']
+ command += ['-S', '.']
+ command += ['-B', 'build']
+ command += ['-G', 'Ninja']
+ command += define_env('CMAKE_BUILD_TYPE')
+ command += define('CMAKE_SUPPRESS_REGENERATION', 'ON')
+ command += define('CMAKE_SKIP_PACKAGE_ALL_DEPENDENCY', 'ON')
+
+ if system() == 'Windows':
+ command += define(
+ 'CMAKE_TOOLCHAIN_FILE',
+ env['VCPKG_DIR'] + '/scripts/buildsystems/vcpkg.cmake'
+ )
+ command += define('VCPKG_TARGET_TRIPLET', env['VCPKG_TRIPLET'])
+ check_call(command)
+
+ command = ['ninja', '-C', 'build']
+ check_call(command)
+
+
+def package(args):
+ command = ['ninja', '-C', 'build', 'package']
+ check_call(command)
+
+
+if __name__ == '__main__':
+ parser = ArgumentParser()
+ subparsers = parser.add_subparsers()
+
+ prepare_parser = subparsers.add_parser('prepare')
+ prepare_parser.set_defaults(task=prepare)
+ prepare_parser.add_argument(
+ '--dependency',
+ dest='dependencies',
+ nargs=4,
+ action='append',
+ metavar=('PACKAGE',
+ 'PROJECT',
+ 'REFERENCE',
+ 'JOB')
+ )
+
+ build_parser = subparsers.add_parser('build')
+ build_parser.set_defaults(task=build)
+
+ package_parser = subparsers.add_parser('package')
+ package_parser.set_defaults(task=package)
+
+ args = parser.parse_args()
+ args.task(args)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ae9bc1..dea1baf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,13 +3,14 @@ stages:
- package
- deploy
+
.ubuntu-18.04:
tags: [ linux, docker ]
image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-ubuntu18.04
-.windows:
- tags: [ windows, docker ]
- image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows
+.windows-1809:
+ tags: [ windows-1809, docker ]
+ image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows1809
.gcc8: &gcc8
@@ -17,65 +18,50 @@ stages:
CXX: g++-8
.msvc15.9: &msvc15_9
- VSDevEnv -arch=amd64 -vcvars_ver="14.16"
-
-.debug: &debug
- CMAKE_BUILD_TYPE: Debug
+ VCVARS_VER: '14.16'
.release: &release
CMAKE_BUILD_TYPE: Release
+.debug: &debug
+ CMAKE_BUILD_TYPE: Debug
-.build ubuntu-18.04:
- extends: .ubuntu-18.04
- stage: build
- artifacts:
- paths:
- - build
- expire_in: 1 day
- script:
- - cmake -Bbuild -H. -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON
- - ninja -C build
-.build windows:
- extends: .windows
+.build: &build
stage: build
artifacts:
paths:
- build
expire_in: 1 day
- script:
- - cmake -Bbuild "-H." -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_DIR/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET="$env:VCPKG_TRIPLET" -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON
- - ninja -C build
+ script: ./.gitlab-ci.py build
build ubuntu-18.04:
- extends: .build ubuntu-18.04
+ extends: .ubuntu-18.04
+ <<: *build
variables:
<<: [ *gcc8, *release ]
before_script:
- - . /etc/profile.d/robofish.sh
- - gitlab-fetch-artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20ubuntu-18.04
- - cmake-integrate-package biotracker-interfaces
+ - ./.gitlab-ci.py prepare
+ --dependency robofish-interfaces bioroboticslab/robofish/interfaces master 'package ubuntu-18.04'
-build windows:
- extends: .build windows
+build windows-1809:
+ extends: .windows-1809
+ <<: *build
variables:
- <<: [ *release ]
+ <<: [ *msvc15_9, *release ]
before_script:
- - . $Profile
- - *msvc15_9
- - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20windows
- - CMake-Integrate-Package biotracker-interfaces
+ - ./.gitlab-ci.py prepare
+ --dependency robofish-interfaces bioroboticslab/robofish/interfaces master 'package windows-1809'
-build windows[debug]:
- extends: .build windows
+build windows-1809[debug]:
+ extends: .windows-1809
+ <<: *build
variables:
- <<: [ *debug ]
+ <<: [ *msvc15_9, *debug ]
before_script:
- - . $Profile
- - *msvc15_9
- - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20windows%5bdebug%5d
- - CMake-Integrate-Package biotracker-interfaces
+ - ./.gitlab-ci.py prepare
+ --dependency robofish-interfaces bioroboticslab/robofish/interfaces master 'package windows-1809[debug]'
+
.package: &package
stage: package
@@ -83,8 +69,7 @@ build windows[debug]:
paths:
- build/*.tar.xz
expire_in: 1 week
- script:
- - ninja -C build package
+ script: ./.gitlab-ci.py package
package ubuntu-18.04:
extends: .ubuntu-18.04
@@ -92,22 +77,20 @@ package ubuntu-18.04:
- build ubuntu-18.04
<<: *package
-package windows:
- extends: .windows
+package windows-1809:
+ extends: .windows-1809
dependencies:
- - build windows
+ - build windows-1809
<<: *package
-package windows[debug]:
- extends: .windows
+package windows-1809[debug]:
+ extends: .windows-1809
dependencies:
- - build windows[debug]
+ - build windows-1809[debug]
<<: *package
-trigger dependents:
- extends: .ubuntu-18.04
- stage: deploy
- script:
- - . /etc/profile.d/robofish.sh
- - gitlab-trigger-pipeline bioroboticslab%2Fbiotracker%2Fbiotracker $CI_JOB_TOKEN master
+trigger robofish/robo_control:
+ stage: deploy
+ trigger:
+ project: bioroboticslab/biotracker/biotracker
--
GitLab