Merge branch 'dev_moritz' into 'master'

Fix #1

Closes #1

See merge request !7

.gitlab-ci.yml

 stages:
-  - test
   - package
+  - test
   - deploy
 
 .centos:
   tags: [linux, docker]
   image: git.imp.fu-berlin.de:5000/bioroboticslab/auto/ci/centos:latest
 
+.macos:
+  tags: [macos, shell]
+
 .windows:
   tags: [windows, docker]
   image: git.imp.fu-berlin.de:5000/bioroboticslab/auto/ci/windows:latest-devel
   before_script:
     - . $Profile.AllUsersAllHosts
 
-.cpy37: &cpy37
-  PYTHON_EXECUTABLE: "python3.7"
-
-.cpy38: &cpy38
-  PYTHON_EXECUTABLE: "python3.8"
+.python38: &python38
+  PYTHON_VERSION: "3.8"
 
 .test: &test
   stage: test
@@ -28,61 +28,53 @@ stages:
   script:
     - ./ci/test.py
 
-test centos[cpy37]:
-  extends: .centos
-  <<: *test
-  variables:
-    <<: [*cpy37]
-
-test centos[cpy38]:
-  extends: .centos
-  <<: *test
-  variables:
-    <<: [*cpy38]
-
-test windows[cpy37]:
-  extends: .windows
-  <<: *test
-
 .package: &package
   stage: package
   artifacts:
     paths:
       - dist
     expire_in: 1 week
+    reports:
+      dotenv: build.env
   script:
     - ./ci/package.py
 
-package centos[cpy37]:
+package:
   extends: .centos
-  dependencies:
-    - test centos[cpy37]
   <<: *package
   variables:
-    <<: [*cpy37]
+    <<: [*python38]
 
-package windows[cpy37]:
+"test: [centos, 3.8]":
+  extends: .centos
+  <<: *test
+
+"test: [macos, 3.8]":
+  extends: .macos
+  <<: *test
+
+"test: [windows, 3.8]":
   extends: .windows
-  dependencies:
-    - test windows[cpy37]
-  <<: *package
+  <<: *test
 
-deploy centos[cpy37]:
+deploy to staging:
   extends: .centos
   stage: deploy
   only:
+    - master
     - tags
+  allow_failure: true
   dependencies:
-    - package centos[cpy37]
+    - package
   script:
-    - ./ci/deploy.py
+    - ./python/ci/deploy.py
 
-.deploy windows[cpy37]:
+deploy to production:
   extends: .centos
   stage: deploy
   only:
     - tags
   dependencies:
-    - package windows[cpy37]
+    - package
   script:
-    - ./ci/deploy.py
+    - ./python/ci/deploy.py --production

ci/deploy.py

@@ -3,22 +3,31 @@
 
 from os import environ as env
 from subprocess import check_call
-from pathlib import Path
 from platform import system
+from argparse import ArgumentParser
 
 
 if __name__ == "__main__":
     if system() != "Linux":
         raise Exception("Uploading python package only supported on Linux")
 
+    p = ArgumentParser()
+    p.add_argument("--production", default=False, action="store_const", const=True)
+    args = p.parse_args()
+
     env["TWINE_USERNAME"] = "gitlab-ci-token"
     env["TWINE_PASSWORD"] = env["CI_JOB_TOKEN"]
 
+    if args.production:
+        target_project_id = env['ARTIFACTS_REPOSITORY_PROJECT_ID']
+    else:
+        target_project_id = env['CI_PROJECT_ID']
+
     command = ["python3"]
     command += ["-m", "twine", "upload", "dist/*"]
     command += [
         "--repository-url",
-        f"https://git.imp.fu-berlin.de/api/v4/projects/{env['ARTIFACTS_REPOSITORY_PROJECT_ID']}/packages/pypi",
+        f"https://git.imp.fu-berlin.de/api/v4/projects/{target_project_id}/packages/pypi",
     ]
 
     check_call(command)

ci/package.py

@@ -2,16 +2,17 @@
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
 from os import environ as env
-from platform import system
 from subprocess import check_call
-from sys import executable
+from pathlib import Path
+from platform import system
+from shutil import which
 
 
 def python_executable():
     if system() == "Windows":
-        return executable
-    elif system() == "Linux":
-        return env["PYTHON_EXECUTABLE"]
+        return f"/Python{''.join(env['PYTHON_VERSION'].split('.'))}/python.exe"
+    elif system() == "Linux" or system() == "Darwin":
+        return which(f"python{env['PYTHON_VERSION']}")
     assert False
 
 
@@ -20,3 +21,6 @@ if __name__ == "__main__":
     command += ["setup.py", "bdist_wheel"]
 
     check_call(command)
+
+    with open("build.env", "w") as f:
+        f.write(f"PYTHON_VERSION={env['PYTHON_VERSION']}\n")

ci/test.py

 #! /usr/bin/env python3
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
-from os import environ as env
+from os import environ as env, pathsep
 from subprocess import check_call
+from pathlib import Path
 from platform import system
-from sys import executable
+from shutil import which
 
 
 def python_executable():
     if system() == "Windows":
-        return executable
-    elif system() == "Linux":
-        return env["PYTHON_EXECUTABLE"]
+        return f"/Python{''.join(env['PYTHON_VERSION'].split('.'))}/python.exe"
+    elif system() == "Linux" or system() == "Darwin":
+        return which(f"python{env['PYTHON_VERSION']}")
+    assert False
+
+
+def python_venv_executable():
+    if system() == "Windows":
+        return str(Path(".venv/Scripts/python.exe").resolve())
+    elif system() == "Linux" or system() == "Darwin":
+        return str(Path(".venv/bin/python").resolve())
     assert False
 
 
 if __name__ == "__main__":
-    check_call([python_executable(), "-m", "pip", "install", "pytest"])
-    # check_call([python_executable(), "-m", "pip", "install", "pytest-cov"])
-    check_call([python_executable(), "-m", "pip", "install", "h5py"])
-    check_call([python_executable(), "-m", "pip", "install", "pandas"])
-    check_call([python_executable(), "-m", "pip", "install", "deprecation"])
+    check_call(
+        [
+            python_executable(),
+            "-m",
+            "venv",
+            "--system-site-packages",
+            "--prompt",
+            "ci",
+            ".venv",
+        ],
+    )
 
-    command = [python_executable()]
-    command += ["-m", "pytest", "--junitxml=report.xml"]
+    check_call(
+        [
+            python_venv_executable(),
+            "-m",
+            "pip",
+            "install",
+            str(sorted(Path("dist").glob("*.whl"))[-1].resolve()),
+        ],
+    )
 
-    check_call(command)
+    check_call(
+        [python_venv_executable(), "-m", "pytest", "--junitxml=report.xml"],
+    )

setup.py

 # SPDX-License-Identifier: LGPL-3.0-or-later
 
+from subprocess import run, PIPE
+
 from setuptools import setup, find_packages
 
 
@@ -11,9 +13,32 @@ entry_points = {
         "robofish-io-evaluate=robofish.evaluate.app:evaluate",
     ]
 }
+
+def source_version():
+    version_parts = (
+        run(["git", "describe", "--tags", "--dirty"], check=True, stdout=PIPE, encoding="utf-8")
+        .stdout.strip()
+        .split("-")
+    )
+
+    if version_parts[-1] == "dirty":
+        dirty = True
+        version_parts = version_parts[:-1]
+    else:
+        dirty = False
+
+    version = version_parts[0]
+    if len(version_parts) == 3:
+        version += ".post0"
+        version += f".dev{version_parts[1]}+{version_parts[2]}"
+    if dirty:
+        version += "+dirty"
+
+    return version
+
 setup(
     name="robofish-io",
-    version="0.1.1",
+    version=source_version(),
     author="",
     author_email="",
     install_requires=["h5py>=3", "numpy", "seaborn", "pandas", "deprecation"],

src/robofish/io/file.py

@@ -28,26 +28,25 @@ import tempfile
 import uuid
 import deprecation
 
-
-temp_dir = tempfile.TemporaryDirectory()
 default_format_version = np.array([1, 0], dtype=np.int32)
 
 default_format_url = (
     "https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/-/releases/1.0"
 )
 
-
 class File(h5py.File):
-    """ Represents a hdf5 file, which should be used to store data about the
-    movement and shape of individuals or swarms in time.
+    """ Represents a RoboFish Track Format file, which should be used to store tracking data of individual animals or swarms.
 
-    This class extends the h5py.File class, and behaves in the same way.
-    robofish io Files can be created, saved, loaded, and manipulated.
+    Files can be opened (with optional creation), modified inplace, and have copies of them saved.
     """
 
+    _temp_dir = None
+
     def __init__(
         self,
         path: Union[str, Path] = None,
+        mode: str = "r",
+        *, # PEP 3102
         world_size_cm: [int, int] = None,
         strict_validate: bool = False,
         format_version: [int, int] = default_format_version,
@@ -57,25 +56,47 @@ class File(h5py.File):
         monotonic_time_points_us: Iterable = None,
         calendar_time_points: Iterable = None,
     ):
-        """ Constructor for the File class.
-
-        The constructor should either be called with a path, when loading an existing file,
-        or when a new file should be created with a world size.
-
-        Args:
-            path: optional path to a io file as a string or path object. The file will be loaded.
-            world_size_cm: optional integer array of the world size in cm
-            strict_validate: optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
-            format_version: optional version [major, minor] of the trackformat specification
+        """Create a new RoboFish Track Format object.
+
+        When called with a path, it is loaded, otherwise a new temporary file is created.
+
+        Parameters
+        ----------
+        path : str or Path, optional
+            Location of file to be opened. If not provided, mode is ignored.
+
+        mode : str
+            r        Readonly, file must exist (default)
+            r+       Read/write, file must exist
+            w        Create file, truncate if exists
+            x        Create file, fail if exists
+            a        Read/write if exists, create otherwise
+
+        world_size_cm
+            optional integer array of the world size in cm
+        strict_validate
+            optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
+        format_version
+            optional version [major, minor] of the trackformat specification
         """
 
-        self._name = str(uuid.uuid4())
-        self._tf_path = Path(temp_dir.name) / self._name
-        self.load(path, strict_validate)
-
-        if path is None or not Path(path).exists():
+        if path is None:
+            if type(self)._temp_dir is None:
+                type(self)._temp_dir = tempfile.TemporaryDirectory(prefix="robofish-io-")
+            super().__init__(Path(type(self)._temp_dir.name) /  str(uuid.uuid4()), mode="x", driver="core", backing_store=True, libver=("earliest", "v112"))
+            initialize = True
+        else:
+            #mode
+            #r        Readonly, file must exist (default)
+            #r+       Read/write, file must exist
+            #w        Create file, truncate if exists
+            #x        Create file, fail if exists
+            #a        Read/write if exists, create otherwise
+            logging.info(f"Opening File {path}")
+            initialize = not Path(path).exists()
+            super().__init__(path, mode, libver=("earliest", "v112"))
 
-            # Initialize new file
+        if initialize:
             assert world_size_cm is not None and format_version is not None
 
             self.attrs["world_size_cm"] = np.array(world_size_cm, dtype=np.float32)
@@ -93,58 +114,32 @@ class File(h5py.File):
                     calendar_time_points=calendar_time_points,
                     default=True,
                 )
+        self.validate(strict_validate)
 
-    #### File Handling ####
-    def load(self, path: Union[str, Path], strict_validate: bool = False) -> None:
-        """ Load a new file from a path
+    def __enter__(self):
+        return self
 
-        Args:
-            path: path to a io file as a string or path object
-            strict_validate: optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
-        """
-        if path is not None:
-            self._f_path = Path(path)
-
-        if path is not None and Path(path).exists():
-            logging.info(f"Opening File {path}")
-            shutil.copyfile(self._f_path, self._tf_path)
-            super().__init__(self._tf_path, "r+")
-            self.validate(strict_validate)
-        else:
-            super().__init__(self._tf_path, "w")
+    def __exit__(self, type, value, traceback):
+        self.validate()
+        self.close()
 
-    def save(self, path: Union[str, Path] = None, strict_validate: bool = True):
-        """ Load a new file from a path
+    def save_as(self, path: Union[str, Path], strict_validate: bool = True):
+        """ Save a copy of the file
 
         Args:
-            path: optional path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
+            path: path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
             strict_validate: optional boolean, if the file should be strictly validated, before saving. The default is True.
         """
 
-        # Normaly only valid files can be saved
         self.validate(strict_validate=strict_validate)
 
-        # Find the correct path
-        if path is None:
-            if self._f_path is None:
-                raise Exception(
-                    "path was not specified and there was no saved path from loading or an earlier save"
-                )
-        else:
-            self._f_path = Path(path)
-
-        # Close the temporary file
-        self.close()
-
-        # Create the parent folder if it does not exist
-        if not self._f_path.parent.exists():
-            self._f_path.parent.mkdir(parents=True, exist_ok=True)
+        # Ensure all buffered data has been written to disk
+        self.flush()
 
-        # Copy the temporaryself.create_sampling(frequency_hz, monotonic_time_points_us) file to the path
-        shutil.copyfile(self._tf_path, self._f_path)
+        path = Path(path).resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
 
-        # Reopen the temporary file
-        super().__init__(self._tf_path, "r+")
+        shutil.copyfile(Path(self.filename).resolve(), path)
 
     def create_sampling(
         self,

tests/robofish/io/test_file.py

@@ -26,10 +26,9 @@ def test_constructor():
 
 def test_new_file_w_path():
     sf = robofish.io.File(
-        created_by_test_path_2, world_size_cm=[100, 100], frequency_hz=25
+        created_by_test_path_2, "w", world_size_cm=[100, 100], frequency_hz=25
     )
     sf.create_entity("fish")
-    sf.save()
     sf.validate()
 
 
@@ -158,7 +157,7 @@ def test_loading_saving():
     sf = test_multiple_entities()
 
     assert not created_by_test_path.exists()
-    sf.save(created_by_test_path)
+    sf.save_as(created_by_test_path)
     assert created_by_test_path.exists()
 
     # After saving, the file should still be accessible and valid