diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 026e854056882ec645610ae2d1a36c21e88b84e4..fd51b975b58349401401f48d90862fe8e7619064 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,23 +1,23 @@
stages:
- - test
- package
+ - test
- deploy
.centos:
tags: [linux, docker]
image: git.imp.fu-berlin.de:5000/bioroboticslab/auto/ci/centos:latest
+.macos:
+ tags: [macos, shell]
+
.windows:
tags: [windows, docker]
image: git.imp.fu-berlin.de:5000/bioroboticslab/auto/ci/windows:latest-devel
before_script:
- . $Profile.AllUsersAllHosts
-.cpy37: &cpy37
- PYTHON_EXECUTABLE: "python3.7"
-
-.cpy38: &cpy38
- PYTHON_EXECUTABLE: "python3.8"
+.python38: &python38
+ PYTHON_VERSION: "3.8"
.test: &test
stage: test
@@ -28,61 +28,53 @@ stages:
script:
- ./ci/test.py
-test centos[cpy37]:
- extends: .centos
- <<: *test
- variables:
- <<: [*cpy37]
-
-test centos[cpy38]:
- extends: .centos
- <<: *test
- variables:
- <<: [*cpy38]
-
-test windows[cpy37]:
- extends: .windows
- <<: *test
-
.package: &package
stage: package
artifacts:
paths:
- dist
expire_in: 1 week
+ reports:
+ dotenv: build.env
script:
- ./ci/package.py
-package centos[cpy37]:
+package:
extends: .centos
- dependencies:
- - test centos[cpy37]
<<: *package
variables:
- <<: [*cpy37]
+ <<: [*python38]
+
+"test: [centos, 3.8]":
+ extends: .centos
+ <<: *test
+
+"test: [macos, 3.8]":
+ extends: .macos
+ <<: *test
-package windows[cpy37]:
+"test: [windows, 3.8]":
extends: .windows
- dependencies:
- - test windows[cpy37]
- <<: *package
+ <<: *test
-deploy centos[cpy37]:
+deploy to staging:
extends: .centos
stage: deploy
only:
+ - master
- tags
+ allow_failure: true
dependencies:
- - package centos[cpy37]
+ - package
script:
- - ./ci/deploy.py
+ - ./python/ci/deploy.py
-.deploy windows[cpy37]:
+deploy to production:
extends: .centos
stage: deploy
only:
- tags
dependencies:
- - package windows[cpy37]
+ - package
script:
- - ./ci/deploy.py
+ - ./python/ci/deploy.py --production
diff --git a/ci/deploy.py b/ci/deploy.py
index 6dda82b345943f9e96b35acb1d1858343e51a1c8..0437457696a996d508cb3b4235ba6feaeb49d52c 100755
--- a/ci/deploy.py
+++ b/ci/deploy.py
@@ -3,22 +3,31 @@
from os import environ as env
from subprocess import check_call
-from pathlib import Path
from platform import system
+from argparse import ArgumentParser
if __name__ == "__main__":
if system() != "Linux":
raise Exception("Uploading python package only supported on Linux")
+ p = ArgumentParser()
+ p.add_argument("--production", default=False, action="store_const", const=True)
+ args = p.parse_args()
+
env["TWINE_USERNAME"] = "gitlab-ci-token"
env["TWINE_PASSWORD"] = env["CI_JOB_TOKEN"]
+ if args.production:
+ target_project_id = env['ARTIFACTS_REPOSITORY_PROJECT_ID']
+ else:
+ target_project_id = env['CI_PROJECT_ID']
+
command = ["python3"]
command += ["-m", "twine", "upload", "dist/*"]
command += [
"--repository-url",
- f"https://git.imp.fu-berlin.de/api/v4/projects/{env['ARTIFACTS_REPOSITORY_PROJECT_ID']}/packages/pypi",
+ f"https://git.imp.fu-berlin.de/api/v4/projects/{target_project_id}/packages/pypi",
]
check_call(command)
diff --git a/ci/package.py b/ci/package.py
index 4be105e3f11445e3c9c8e9d183882fcac116d8ba..38ef2d7f540eb37068ae0a0df5123173962e2789 100755
--- a/ci/package.py
+++ b/ci/package.py
@@ -2,16 +2,17 @@
# SPDX-License-Identifier: LGPL-3.0-or-later
from os import environ as env
-from platform import system
from subprocess import check_call
-from sys import executable
+from pathlib import Path
+from platform import system
+from shutil import which
def python_executable():
if system() == "Windows":
- return executable
- elif system() == "Linux":
- return env["PYTHON_EXECUTABLE"]
+ return f"/Python{''.join(env['PYTHON_VERSION'].split('.'))}/python.exe"
+ elif system() == "Linux" or system() == "Darwin":
+ return which(f"python{env['PYTHON_VERSION']}")
assert False
@@ -20,3 +21,6 @@ if __name__ == "__main__":
command += ["setup.py", "bdist_wheel"]
check_call(command)
+
+ with open("build.env", "w") as f:
+ f.write(f"PYTHON_VERSION={env['PYTHON_VERSION']}\n")
diff --git a/ci/test.py b/ci/test.py
index 992a7cf573a716ff839a26989fd45dab6ce66c6c..c6d90a39d599fc668a6a1ed525c2a940ddeeec4f 100755
--- a/ci/test.py
+++ b/ci/test.py
@@ -1,28 +1,52 @@
#! /usr/bin/env python3
# SPDX-License-Identifier: LGPL-3.0-or-later
-from os import environ as env
+from os import environ as env, pathsep
from subprocess import check_call
+from pathlib import Path
from platform import system
-from sys import executable
+from shutil import which
def python_executable():
if system() == "Windows":
- return executable
- elif system() == "Linux":
- return env["PYTHON_EXECUTABLE"]
+ return f"/Python{''.join(env['PYTHON_VERSION'].split('.'))}/python.exe"
+ elif system() == "Linux" or system() == "Darwin":
+ return which(f"python{env['PYTHON_VERSION']}")
+ assert False
+
+
+def python_venv_executable():
+ if system() == "Windows":
+ return str(Path(".venv/Scripts/python.exe").resolve())
+ elif system() == "Linux" or system() == "Darwin":
+ return str(Path(".venv/bin/python").resolve())
assert False
if __name__ == "__main__":
- check_call([python_executable(), "-m", "pip", "install", "pytest"])
- # check_call([python_executable(), "-m", "pip", "install", "pytest-cov"])
- check_call([python_executable(), "-m", "pip", "install", "h5py"])
- check_call([python_executable(), "-m", "pip", "install", "pandas"])
- check_call([python_executable(), "-m", "pip", "install", "deprecation"])
+ check_call(
+ [
+ python_executable(),
+ "-m",
+ "venv",
+ "--system-site-packages",
+ "--prompt",
+ "ci",
+ ".venv",
+ ],
+ )
- command = [python_executable()]
- command += ["-m", "pytest", "--junitxml=report.xml"]
+ check_call(
+ [
+ python_venv_executable(),
+ "-m",
+ "pip",
+ "install",
+ str(sorted(Path("dist").glob("*.whl"))[-1].resolve()),
+ ],
+ )
- check_call(command)
+ check_call(
+ [python_venv_executable(), "-m", "pytest", "--junitxml=report.xml"],
+ )
diff --git a/setup.py b/setup.py
index 456a82480daf446983066cb830724c7fcc11548e..5be2fe4573c17f9dbb776058058ea0e38c4674aa 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: LGPL-3.0-or-later
+from subprocess import run, PIPE
+
from setuptools import setup, find_packages
@@ -11,9 +13,32 @@ entry_points = {
"robofish-io-evaluate=robofish.evaluate.app:evaluate",
]
}
+
+def source_version():
+ version_parts = (
+ run(["git", "describe", "--tags", "--dirty"], check=True, stdout=PIPE, encoding="utf-8")
+ .stdout.strip()
+ .split("-")
+ )
+
+ if version_parts[-1] == "dirty":
+ dirty = True
+ version_parts = version_parts[:-1]
+ else:
+ dirty = False
+
+ version = version_parts[0]
+ if len(version_parts) == 3:
+ version += ".post0"
+ version += f".dev{version_parts[1]}+{version_parts[2]}"
+ if dirty:
+ version += "+dirty"
+
+ return version
+
setup(
name="robofish-io",
- version="0.1.1",
+ version=source_version(),
author="",
author_email="",
install_requires=["h5py>=3", "numpy", "seaborn", "pandas", "deprecation"],
diff --git a/src/robofish/io/file.py b/src/robofish/io/file.py
index deefcc9a771d6e54966224be4be1dc1abbb69c16..1914f99d520facecaeb122da8a83b2ac86cb48be 100644
--- a/src/robofish/io/file.py
+++ b/src/robofish/io/file.py
@@ -28,26 +28,25 @@ import tempfile
import uuid
import deprecation
-
-temp_dir = tempfile.TemporaryDirectory()
default_format_version = np.array([1, 0], dtype=np.int32)
default_format_url = (
"https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/-/releases/1.0"
)
-
class File(h5py.File):
- """ Represents a hdf5 file, which should be used to store data about the
- movement and shape of individuals or swarms in time.
+ """ Represents a RoboFish Track Format file, which should be used to store tracking data of individual animals or swarms.
- This class extends the h5py.File class, and behaves in the same way.
- robofish io Files can be created, saved, loaded, and manipulated.
+ Files can be opened (with optional creation), modified inplace, and have copies of them saved.
"""
+ _temp_dir = None
+
def __init__(
self,
path: Union[str, Path] = None,
+ mode: str = "r",
+ *, # PEP 3102
world_size_cm: [int, int] = None,
strict_validate: bool = False,
format_version: [int, int] = default_format_version,
@@ -57,25 +56,47 @@ class File(h5py.File):
monotonic_time_points_us: Iterable = None,
calendar_time_points: Iterable = None,
):
- """ Constructor for the File class.
-
- The constructor should either be called with a path, when loading an existing file,
- or when a new file should be created with a world size.
-
- Args:
- path: optional path to a io file as a string or path object. The file will be loaded.
- world_size_cm: optional integer array of the world size in cm
- strict_validate: optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
- format_version: optional version [major, minor] of the trackformat specification
+ """Create a new RoboFish Track Format object.
+
+ When called with a path, it is loaded, otherwise a new temporary file is created.
+
+ Parameters
+ ----------
+ path : str or Path, optional
+ Location of file to be opened. If not provided, mode is ignored.
+
+ mode : str
+ r Readonly, file must exist (default)
+ r+ Read/write, file must exist
+ w Create file, truncate if exists
+ x Create file, fail if exists
+ a Read/write if exists, create otherwise
+
+ world_size_cm
+ optional integer array of the world size in cm
+ strict_validate
+ optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
+ format_version
+ optional version [major, minor] of the trackformat specification
"""
- self._name = str(uuid.uuid4())
- self._tf_path = Path(temp_dir.name) / self._name
- self.load(path, strict_validate)
-
- if path is None or not Path(path).exists():
+ if path is None:
+ if type(self)._temp_dir is None:
+ type(self)._temp_dir = tempfile.TemporaryDirectory(prefix="robofish-io-")
+ super().__init__(Path(type(self)._temp_dir.name) / str(uuid.uuid4()), mode="x", driver="core", backing_store=True, libver=("earliest", "v112"))
+ initialize = True
+ else:
+ #mode
+ #r Readonly, file must exist (default)
+ #r+ Read/write, file must exist
+ #w Create file, truncate if exists
+ #x Create file, fail if exists
+ #a Read/write if exists, create otherwise
+ logging.info(f"Opening File {path}")
+ initialize = not Path(path).exists()
+ super().__init__(path, mode, libver=("earliest", "v112"))
- # Initialize new file
+ if initialize:
assert world_size_cm is not None and format_version is not None
self.attrs["world_size_cm"] = np.array(world_size_cm, dtype=np.float32)
@@ -93,58 +114,32 @@ class File(h5py.File):
calendar_time_points=calendar_time_points,
default=True,
)
+ self.validate(strict_validate)
- #### File Handling ####
- def load(self, path: Union[str, Path], strict_validate: bool = False) -> None:
- """ Load a new file from a path
+ def __enter__(self):
+ return self
- Args:
- path: path to a io file as a string or path object
- strict_validate: optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
- """
- if path is not None:
- self._f_path = Path(path)
-
- if path is not None and Path(path).exists():
- logging.info(f"Opening File {path}")
- shutil.copyfile(self._f_path, self._tf_path)
- super().__init__(self._tf_path, "r+")
- self.validate(strict_validate)
- else:
- super().__init__(self._tf_path, "w")
+ def __exit__(self, type, value, traceback):
+ self.validate()
+ self.close()
- def save(self, path: Union[str, Path] = None, strict_validate: bool = True):
- """ Load a new file from a path
+ def save_as(self, path: Union[str, Path], strict_validate: bool = True):
+ """ Save a copy of the file
Args:
- path: optional path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
+ path: path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
strict_validate: optional boolean, if the file should be strictly validated, before saving. The default is True.
"""
- # Normaly only valid files can be saved
self.validate(strict_validate=strict_validate)
- # Find the correct path
- if path is None:
- if self._f_path is None:
- raise Exception(
- "path was not specified and there was no saved path from loading or an earlier save"
- )
- else:
- self._f_path = Path(path)
-
- # Close the temporary file
- self.close()
-
- # Create the parent folder if it does not exist
- if not self._f_path.parent.exists():
- self._f_path.parent.mkdir(parents=True, exist_ok=True)
+ # Ensure all buffered data has been written to disk
+ self.flush()
- # Copy the temporaryself.create_sampling(frequency_hz, monotonic_time_points_us) file to the path
- shutil.copyfile(self._tf_path, self._f_path)
+ path = Path(path).resolve()
+ path.parent.mkdir(parents=True, exist_ok=True)
- # Reopen the temporary file
- super().__init__(self._tf_path, "r+")
+ shutil.copyfile(Path(self.filename).resolve(), path)
def create_sampling(
self,
diff --git a/tests/robofish/io/test_file.py b/tests/robofish/io/test_file.py
index 3cb5d023681937c8d9334a6017de294820e2a128..8ce4bd4b9899f0fd785dcf315e7f75ee79eb26e2 100644
--- a/tests/robofish/io/test_file.py
+++ b/tests/robofish/io/test_file.py
@@ -26,10 +26,9 @@ def test_constructor():
def test_new_file_w_path():
sf = robofish.io.File(
- created_by_test_path_2, world_size_cm=[100, 100], frequency_hz=25
+ created_by_test_path_2, "w", world_size_cm=[100, 100], frequency_hz=25
)
sf.create_entity("fish")
- sf.save()
sf.validate()
@@ -158,7 +157,7 @@ def test_loading_saving():
sf = test_multiple_entities()
assert not created_by_test_path.exists()
- sf.save(created_by_test_path)
+ sf.save_as(created_by_test_path)
assert created_by_test_path.exists()
# After saving, the file should still be accessible and valid