Update ci/cd

.gitlab-ci.py

+#! /usr/bin/env python3
+
+import tarfile
+import shutil
+import ssl
+
+from os import environ as env
+from platform import system
+from subprocess import check_call, check_output
+from argparse import ArgumentParser
+from io import BytesIO
+from urllib.request import Request, urlopen
+from urllib.parse import quote, urlencode
+from zipfile import ZipFile
+from pathlib import Path
+
+
+def define(key: str, value: str):
+    return ['-D', f'{key}={value}']
+
+
+def define_env(name: str):
+    return define(name, env[name]) if name in env else []
+
+
+def fetch_artifacts(project, reference, job):
+    gitlab_host = 'https://git.imp.fu-berlin.de'
+    project = quote(project, safe="")
+    reference = quote(reference, safe="")
+    params = urlencode([("job", job)], doseq=True)
+    url = f'{gitlab_host}/api/v4/projects/{project}/jobs/artifacts/{reference}/download?{params}'
+    headers = {'JOB-TOKEN': env['CI_JOB_TOKEN']}
+    return ZipFile(
+        BytesIO(
+            urlopen(Request(url, headers=headers),
+                    context=ssl._create_unverified_context()).read()))
+
+
+def extract_cmake_package(artifacts, name):
+    for filename in artifacts.namelist():
+        if Path(filename).match(f'{name}-*.tar.xz'):
+            with tarfile.open(fileobj=BytesIO(artifacts.read(filename))) as f:
+                f.extractall('vendor')
+            shutil.move(next(Path('vendor').glob(f'{name}-*/')),
+                        f'vendor/{name}')
+
+
+if system() == 'Windows':
+
+    def setup_msvc():
+        msvc_path = 'C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/Tools'
+        lines = check_output([
+            'cmd', '/c', 'VsDevCmd.bat', '-arch=amd64',
+            f'-vcvars_ver={env["VCVARS_VER"]}', '&', 'set'
+        ],
+                             cwd=msvc_path).decode('utf-8').splitlines()
+        for line in lines:
+            split = line.split('=')
+            if len(split) != 2:
+                continue
+            key, value = split
+            if key in env and env[key] == value:
+                continue
+            env[key] = value
+
+
+def prepare(args):
+    for name, project, reference, job in args.dependencies:
+        with fetch_artifacts(project, reference, job) as artifacts:
+            extract_cmake_package(artifacts, name)
+
+
+def build(args):
+    if system() == 'Windows':
+        setup_msvc()
+
+    command = ['cmake']
+    command += ['-S', '.']
+    command += ['-B', 'build']
+    command += ['-G', 'Ninja']
+    command += define('CMAKE_PREFIX_PATH', 'vendor')
+    command += define_env('CMAKE_BUILD_TYPE')
+    command += define('CMAKE_SUPPRESS_REGENERATION', 'ON')
+    command += define('CMAKE_SKIP_PACKAGE_ALL_DEPENDENCY', 'ON')
+
+    if system() == 'Windows':
+        command += define(
+            'CMAKE_TOOLCHAIN_FILE',
+            env['VCPKG_DIR'] + '/scripts/buildsystems/vcpkg.cmake')
+        command += define('VCPKG_TARGET_TRIPLET', env['VCPKG_TRIPLET'])
+        command += define('PACKAGE_WINDOWS_MSI', 'ON')
+    elif system() == 'Linux':
+        command += define('PACKAGE_TXZ', 'ON')
+    check_call(command)
+
+    command = ['ninja', '-C', 'build']
+    check_call(command)
+
+
+def package(args):
+    command = ['ninja', '-C', 'build', 'package']
+    check_call(command)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    prepare_parser = subparsers.add_parser('prepare')
+    prepare_parser.set_defaults(task=prepare)
+    prepare_parser.add_argument('--dependency',
+                                dest='dependencies',
+                                nargs=4,
+                                action='append',
+                                metavar=('PACKAGE', 'PROJECT', 'REFERENCE',
+                                         'JOB'))
+
+    build_parser = subparsers.add_parser('build')
+    build_parser.set_defaults(task=build)
+
+    package_parser = subparsers.add_parser('package')
+    package_parser.set_defaults(task=package)
+
+    args = parser.parse_args()
+    args.task(args)

.gitlab-ci.yml

@@ -3,13 +3,14 @@ stages:
   - package
   - deploy
 
+
 .ubuntu-18.04:
   tags: [ linux, docker ]
   image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-ubuntu18.04
 
-.windows:
-  tags: [ windows, docker ]
-  image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows
+.windows-1809:
+  tags: [ windows-1809, docker ]
+  image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows1809
 
 
 .gcc8: &gcc8
@@ -17,60 +18,44 @@ stages:
   CXX: g++-8
 
 .msvc15.9: &msvc15_9
-  VSDevEnv -arch=amd64 -vcvars_ver="14.16"
-
-.debug: &debug
-  CMAKE_BUILD_TYPE: Debug
+  VCVARS_VER: '14.16'
 
 .release: &release
   CMAKE_BUILD_TYPE: Release
 
+.debug: &debug
+  CMAKE_BUILD_TYPE: Debug
 
-.build ubuntu-18.04:
-  extends: .ubuntu-18.04
-  stage: build
-  artifacts:
-    paths:
-      - build
-    expire_in: 1 day
-  script:
-    - cmake -Bbuild -H. -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DPACKAGE_TXZ=ON -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON
-    - ninja -C build
 
-.build windows:
-  extends: .windows
+.build: &build
   stage: build
   artifacts:
     paths:
+      - vendor
       - build
     expire_in: 1 day
-  script:
-    - cmake -Bbuild "-H." -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -G Ninja -DPACKAGE_WINDOWS_MSI=ON -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_DIR/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET="$env:VCPKG_TRIPLET" -DCMAKE_SUPPRESS_REGENERATION=ON -DCMAKE_SKIP_PACKAGE_ALL_DEPENDENCY=ON -DCPACK_PACKAGE_DIRECTORY="${env:PACKAGE_DIR}"
-    - ninja -C build
+  script: ./.gitlab-ci.py build
 
 build ubuntu-18.04:
-  extends: .build ubuntu-18.04
+  extends: .ubuntu-18.04
+  <<: *build
   variables:
     <<: [ *gcc8, *release ]
   before_script:
-    - . /etc/profile.d/robofish.sh
-    - gitlab-fetch-artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20ubuntu-18.04
-    - cmake-integrate-package biotracker-interfaces
-    - gitlab-fetch-artifacts bioroboticslab%2Fbiotracker%2Futility master package%20ubuntu-18.04
-    - cmake-integrate-package biotracker-utility
-
-build windows:
-  extends: .build windows
+    - ./.gitlab-ci.py prepare
+      --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package ubuntu-18.04'
+      --dependency biotracker-utility    bioroboticslab/biotracker/utility    master 'package ubuntu-18.04'
+
+build windows-1809:
+  extends: .windows-1809
+  <<: *build
   variables:
-    <<: [ *release ]
+    <<: [ *msvc15_9, *release ]
   before_script:
-    - . $Profile
-    - *msvc15_9
-    - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Finterfaces master package%20windows
-    - CMake-Integrate-Package biotracker-interfaces
-    - GitLab-Fetch-Artifacts bioroboticslab%2Fbiotracker%2Futility master package%20windows
-    - CMake-Integrate-Package biotracker-utility
-    - $env:PACKAGE_DIR = ("${env:TMP}/pkg" -Replace "\\","/")
+    - ./.gitlab-ci.py prepare
+      --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package windows-1809'
+      --dependency biotracker-utility    bioroboticslab/biotracker/utility    master 'package windows-1809'
+
 
 .package: &package
   stage: package
@@ -79,8 +64,7 @@ build windows:
       - build/*.tar.xz
       - build/*.msi
     expire_in: 1 week
-  script:
-    - ninja -C build package
+  script: ./.gitlab-ci.py package
 
 package ubuntu-18.04:
   extends: .ubuntu-18.04
@@ -88,13 +72,8 @@ package ubuntu-18.04:
     - build ubuntu-18.04
   <<: *package
 
-package windows:
-  extends: .windows
+package windows-1809:
+  extends: .windows-1809
   dependencies:
-    - build windows
+    - build windows-1809
   <<: *package
-  before_script:
-    - $env:PACKAGE_DIR = ("${env:TMP}/pkg" -Replace "\\","/")
-  script:
-    - ninja -C build package
-    - Copy-Item "$env:PACKAGE_DIR/*" build -Recurse