diff --git a/.gitlab-ci.py b/.gitlab-ci.py deleted file mode 100755 index 25f3da0ca4c4c1ef200b14ee156ac827ae8c6abb..0000000000000000000000000000000000000000 --- a/.gitlab-ci.py +++ /dev/null @@ -1,135 +0,0 @@ -#! /usr/bin/env python3 - -import tarfile -import shutil -import ssl - -from os import environ as env, symlink -from platform import system -from subprocess import check_call, check_output, DEVNULL -from argparse import ArgumentParser -from io import BytesIO -from urllib.request import Request, urlopen -from urllib.parse import quote, urlencode -from zipfile import ZipFile -from pathlib import Path - - -def define(key: str, value: str): - return ['-D', f'{key}={value}'] - - -def define_env(name: str): - return define(name, env[name]) if name in env else [] - - -def fetch_artifacts(project, reference, job): - gitlab_host = 'https://git.imp.fu-berlin.de' - project = quote(project, safe="") - reference = quote(reference, safe="") - params = urlencode([("job", job)], doseq=True) - url = f'{gitlab_host}/api/v4/projects/{project}/jobs/artifacts/{reference}/download?{params}' - headers = {'JOB-TOKEN': env['CI_JOB_TOKEN']} - return ZipFile( - BytesIO( - urlopen(Request(url, headers=headers), - context=ssl._create_unverified_context()).read())) - - -def extract_cmake_package(artifacts, name): - for filename in artifacts.namelist(): - if Path(filename).match(f'{name}-*.tar.xz'): - with tarfile.open(fileobj=BytesIO(artifacts.read(filename))) as f: - f.extractall('vendor') - shutil.move(next(Path('vendor').glob(f'{name}-*/')), - f'vendor/{name}') - - -def enable_cmake_package_discovery(name): - if system() == 'Windows': - check_call([ - 'powershell', '&', 'cmd.exe', '/c', 'mklink', '/J', - f'"C:/Program Files/{name}"', f'"$(resolve-path vendor/{name})"' - ], - stdout=DEVNULL) - elif system() == 'Linux': - symlink(Path(f'vendor/{name}').resolve(), f'/usr/local/{name}') - else: - assert False - - -if system() == 'Windows': - - def setup_msvc(): - msvc_path = 'C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/Tools' - lines = check_output([ - 'cmd', '/c', 'VsDevCmd.bat', '-arch=amd64', - f'-vcvars_ver={env["VCVARS_VER"]}', '&', 'set' - ], - cwd=msvc_path).decode('utf-8').splitlines() - for line in lines: - split = line.split('=') - if len(split) != 2: - continue - key, value = split - if key in env and env[key] == value: - continue - env[key] = value - - -def prepare(args): - for name, project, reference, job in args.dependencies: - with fetch_artifacts(project, reference, job) as artifacts: - extract_cmake_package(artifacts, name) - - -def build(args): - if system() == 'Windows': - setup_msvc() - - command = ['cmake'] - command += ['-S', '.'] - command += ['-B', 'build'] - command += ['-G', 'Ninja'] - command += define('CMAKE_PREFIX_PATH', Path('vendor').resolve()) - command += define_env('CMAKE_BUILD_TYPE') - command += define('CMAKE_SUPPRESS_REGENERATION', 'ON') - command += define('CMAKE_SKIP_PACKAGE_ALL_DEPENDENCY', 'ON') - - if system() == 'Windows': - command += define( - 'CMAKE_TOOLCHAIN_FILE', - env['VCPKG_DIR'] + '/scripts/buildsystems/vcpkg.cmake') - command += define('VCPKG_TARGET_TRIPLET', env['VCPKG_TRIPLET']) - check_call(command) - - command = ['ninja', '-C', 'build'] - check_call(command) - - -def package(args): - command = ['ninja', '-C', 'build', 'package'] - check_call(command) - - -if __name__ == '__main__': - parser = ArgumentParser() - subparsers = parser.add_subparsers() - - prepare_parser = subparsers.add_parser('prepare') - prepare_parser.set_defaults(task=prepare) - prepare_parser.add_argument('--dependency', - dest='dependencies', - nargs=4, - action='append', - metavar=('PACKAGE', 'PROJECT', 'REFERENCE', - 'JOB')) - - build_parser = subparsers.add_parser('build') - build_parser.set_defaults(task=build) - - package_parser = subparsers.add_parser('package') - package_parser.set_defaults(task=package) - - args = parser.parse_args() - args.task(args) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a954aa40e83dfaea8168c395342f8c627f7b2c0..abe466080b531f979feb696bf57f90bd64899c7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,66 +1,31 @@ stages: - build - package - - deploy + - trigger -.centos-7: - tags: [ linux, docker ] - image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:centos-7 +.centos: + tags: [linux, docker] + image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:centos -.windows-1809: - tags: [ windows-1809, docker ] - image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows-1809 - - -.gcc8: &gcc8 - CC: gcc-8 - CXX: g++-8 - -.msvc15.9: &msvc15_9 - VCVARS_VER: '14.16' +.windows: + tags: [windows, docker] + image: git.imp.fu-berlin.de:5000/bioroboticslab/robofish/docker:devel-windows + before_script: + - . $Profile.AllUsersAllHosts .release: &release CMAKE_BUILD_TYPE: Release -.debug: &debug - CMAKE_BUILD_TYPE: Debug - - .build: &build stage: build artifacts: paths: - build expire_in: 1 day - script: ./.gitlab-ci.py build - -build centos-7: - extends: .centos-7 - <<: *build - variables: - <<: [ *release ] - before_script: - - ./.gitlab-ci.py prepare - --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package centos-7' - -build windows-1809: - extends: .windows-1809 - <<: *build - variables: - <<: [ *msvc15_9, *release ] - before_script: - - ./.gitlab-ci.py prepare - --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package windows-1809' - -build windows-1809[debug]: - extends: .windows-1809 - <<: *build - variables: - <<: [ *msvc15_9, *debug ] - before_script: - - ./.gitlab-ci.py prepare - --dependency biotracker-interfaces bioroboticslab/biotracker/interfaces master 'package windows-1809[debug]' - + script: + - ./ci/prepare.py + - ./ci/configure.py + - ./ci/compile.py .package: &package stage: package @@ -68,28 +33,36 @@ build windows-1809[debug]: paths: - build/*.tar.xz expire_in: 1 week - script: ./.gitlab-ci.py package + script: + - ./ci/package.py -package centos-7: - extends: .centos-7 - dependencies: - - build centos-7 - <<: *package +build centos: + extends: .centos + <<: *build + variables: + <<: [*release] + +build windows: + extends: .windows + <<: *build + variables: + <<: [*release] -package windows-1809: - extends: .windows-1809 +package centos: + extends: .centos dependencies: - - build windows-1809 + - build centos <<: *package -package windows-1809[debug]: - extends: .windows-1809 +package windows: + extends: .windows dependencies: - - build windows-1809[debug] + - build windows <<: *package - trigger biotracker/biotracker: - stage: deploy + stage: trigger + only: + - master trigger: project: bioroboticslab/biotracker/biotracker diff --git a/ci/compile.py b/ci/compile.py new file mode 100755 index 0000000000000000000000000000000000000000..bf791920d530d414a659f6ea7249d4baad6aaa92 --- /dev/null +++ b/ci/compile.py @@ -0,0 +1,7 @@ +#! /usr/bin/env python3 + +from subprocess import check_call + +if __name__ == "__main__": + command = ["ninja", "-C", "build"] + check_call(command) diff --git a/ci/configure.py b/ci/configure.py new file mode 100755 index 0000000000000000000000000000000000000000..bdecb85a72a85d6e5b6b7cbadd6f6682dbbb1798 --- /dev/null +++ b/ci/configure.py @@ -0,0 +1,28 @@ +#! /usr/bin/env python3 + +from os import environ as env +from subprocess import check_call +from pathlib import Path + + +def define(key: str, value: str): + return ["-D", f"{key}={value}"] + + +def define_env(name: str): + return define(name, env[name]) if name in env else [] + + +if __name__ == "__main__": + command = ["cmake"] + command += ["-S", "."] + command += ["-B", "build"] + command += ["-G", "Ninja"] + command += define('CMAKE_PREFIX_PATH', Path('vendor').resolve()) + command += define_env("CMAKE_BUILD_TYPE") + command += define_env("CMAKE_TOOLCHAIN_FILE") + command += define("CMAKE_SUPPRESS_REGENERATION", "ON") + command += define("CMAKE_SKIP_PACKAGE_ALL_DEPENDENCY", "ON") + command += define_env("VCPKG_TARGET_TRIPLET") + + check_call(command) diff --git a/ci/package.py b/ci/package.py new file mode 100755 index 0000000000000000000000000000000000000000..fe4032f6e45cc938f14f7e4dac148a6709afea22 --- /dev/null +++ b/ci/package.py @@ -0,0 +1,7 @@ +#! /usr/bin/env python3 + +from subprocess import check_call + +if __name__ == "__main__": + command = ["ninja", "-C", "build", "package"] + check_call(command) diff --git a/ci/prepare.py b/ci/prepare.py new file mode 100755 index 0000000000000000000000000000000000000000..02203017a195883429a9b1e9ad544dace7b4ed99 --- /dev/null +++ b/ci/prepare.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python3 + +import tarfile +import shutil +import ssl + +from os import environ as env, symlink +from platform import system +from io import BytesIO +from urllib.request import Request, urlopen +from urllib.parse import quote, urlencode +from zipfile import ZipFile +from pathlib import Path + + +def define(key: str, value: str): + return ["-D", f"{key}={value}"] + + +def define_env(name: str): + return define(name, env[name]) if name in env else [] + + +def fetch_artifacts(project, reference, job): + gitlab_host = "https://git.imp.fu-berlin.de" + project = quote(project, safe="") + reference = quote(reference, safe="") + params = urlencode([("job", job)], doseq=True) + url = f"{gitlab_host}/api/v4/projects/{project}/jobs/artifacts/{reference}/download?{params}" + headers = {"JOB-TOKEN": env["CI_JOB_TOKEN"]} + return ZipFile( + BytesIO( + urlopen( + Request(url, headers=headers), context=ssl._create_unverified_context() + ).read() + ) + ) + + +def extract_cmake_package(artifacts, name): + for filename in artifacts.namelist(): + if Path(filename).match(f"{name}-*.tar.xz"): + with tarfile.open(fileobj=BytesIO(artifacts.read(filename))) as f: + f.extractall("vendor") + shutil.move(next(Path("vendor").glob(f"{name}-*/")), f"vendor/{name}") + + +if __name__ == "__main__": + if system() == "Windows": + job_stem = "package windows" + elif system() == "Linux": + job_stem = "package centos" + else: + assert False + + for name, project, job in [ + ("biotracker-interfaces", "bioroboticslab/biotracker/interfaces", job_stem), + ]: + with fetch_artifacts(project, "master", job) as artifacts: + extract_cmake_package(artifacts, name)