Merge branch 'master' into 'develop_marc'

# Conflicts:
#   src/robofish/evaluate/evaluate.py

.gitignore

@@ -8,7 +8,7 @@ dist
 .coverage
 report.xml
 htmlcov
-docs
+html
 env
 !tests/resources/*.hdf5
 

README.md

@@ -9,59 +9,60 @@
 [![pipeline status](https://git.imp.fu-berlin.de/bioroboticslab/robofish/io/badges/master/pipeline.svg)](https://git.imp.fu-berlin.de/bioroboticslab/robofish/io/commits/master)
 # Robofish IO
 
-This repository implements an easy to use interface, to create, save, load, and work  [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files.
+This repository implements an easy to use interface, to create, save, load, and work with [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files.
 
+## <a href="http://agerken.de/io/index.html">Documentation</a>
 
 ## Installation
 
-Quick variant:
-```
-pip3 install robofish-trackviewer robofish-io --extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple
-```
-
-Better variant:
-- Follow instructions at [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts)
-- ```pip3 install robofish-io```
+Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) to your pip config and install the packagage.
 
+```bash
+python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple
+python3 -m pip install robofish-io
+```
 
 ## Usage
 We show a simple example below. More examples can be found in ```examples/```
 
 ```python
-import robofish.io
-import numpy as np
-
-
 # Create a new robofish io file
 f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
 f.attrs["experiment_setup"] = "This is a simple example with made up data."
 
-# Create a new robot entity. Positions and orientations are passed
-# separately in this example. Since the orientations have two columns,
-# unit vectors are assumed (orientation_x, orientation_y)
+# Create a new robot entity with 10 timesteps.
+# Positions and orientations are passed separately in this example.
+# Since the orientations have two columns, unit vectors are assumed
+# (orientation_x, orientation_y)
 f.create_entity(
     category="robot",
     name="robot",
-    positions=np.zeros((100, 2)),
-    orientations=np.ones((100, 2)) * [0, 1],
+    positions=np.zeros((10, 2)),
+    orientations=np.ones((10, 2)) * [0, 1],
 )
 
-# Create a new fish entity.
+# Create a new fish entity with 10 timesteps.
 # In this case, we pass positions and orientations together (x, y, rad).
 # Since it is a 3 column array, orientations in radiants are assumed.
-poses = np.zeros((100, 3))
-poses[:, 0] = np.arange(-50, 50)
-poses[:, 1] = np.arange(-50, 50)
-poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100)
+poses = np.zeros((10, 3))
+poses[:, 0] = np.arange(-5, 5)
+poses[:, 1] = np.arange(-5, 5)
+poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
 fish = f.create_entity("fish", poses=poses)
 fish.attrs["species"] = "My rotating spaghetti fish"
 fish.attrs["fish_standard_length_cm"] = 10
 
-# Show and save the file
-print(f)
-print("Poses Shape: ", f.entity_poses.shape)
+# Some possibilities to access the data
+print(f"The file:\n{f}")
+print(
+    f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+    + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+)
+print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
 
-f.save_as(path)
+# Save the file
+f.save_as("example.hdf5")
 ```
 
 ### Evaluation
@@ -72,13 +73,3 @@ Current modes are:
 - tank_positions
 - trajectories
 - follow_iid
\ No newline at end of file
-
-
-## LICENSE
-
-This work is licensed under LGPL 3.0 (or any later version).
-Individual files contain the following tag instead of the full license text:
-
-`SPDX-License-Identifier: LGPL-3.0-or-later`
-
-This enables machine processing of license information based on the SPDX License Identifiers available here: https://spdx.org/licenses/

docs/app.md

+This module defines the command line interface.
+All commands have a ``--help`` option to get more info about them in the command line.
+
+
+## Print
+```bash
+robofish-io-print example.hdf5
+```
+Checking out the content of files.
+
+<small>
+
+```
+usage: robofish-io-print [-h] [--output_format {shape,full}] path
+
+This function can be used to print hdf5 files from the command line
+positional arguments:
+  path                  The path to a hdf5 file
+optional arguments:
+  -h, --help            show this help message and exit
+  --output_format {shape,full}
+                        Choose how datasets are printed, either the shapes or the full content is printed
+```
+
+</small>
+
+## Evaluate
+```bash
+robofish-io-evaluate *analysis_type* example.hdf5
+```
+Show some property of a file.
+
+
+<small>
+
+```
+usage: robofish-io-evaluate [-h] [--names NAMES [NAMES ...]] [--save_path SAVE_PATH]
+                            {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid} paths [paths ...]
+
+This function can be called from the commandline to evaluate files.
+Different evaluation methods can be called, which generate graphs from the given files.
+With the first argument 'analysis_type', the type of analysis is chosen.
+
+positional arguments:
+  {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid}
+                        The type of analysis.
+                        speed                - Evaluate the speed of the entities as histogram.
+                        turn                 - Evaluate the turn angles of the entities as histogram.
+                        orientation          - Evaluate the orientations of the entities on a 2d grid.
+                        relative_orientation - Evaluate the relative orientations of the entities as a histogram.
+                        distance_to_wall     - Evaluate the distances of the entities to the walls as a histogram.
+                        tank_positions       - Evaluate the positions of the entities as a heatmap.
+                        trajectories         - Evaluate the trajectories of the entities.
+                        evaluate_positionVec - Evaluate the vectors pointing from the focal fish to the conspecifics as heatmap.
+                        follow_iid           - Evaluate the follow metric in respect to the inter individual distance (iid).
+  paths                 The paths to files or folders. Multiple paths can be given to compare experiments.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --names NAMES [NAMES ...]
+                        Names, that should be used in the graphs instead of the pahts.
+  --save_path SAVE_PATH
+                        Filename for saving resulting graphics.
+```
+
+</small>
+
+## Trackviewer
+
+```bash
+robofish-trackviewer example.hdf5
+```
+
+The trackviewer is from a different repository. It was included in the install instructions.
+
+<small>
+
+```
+usage: robofish-trackviewer [-h] [--draw-labels] [--draw-view-vectors] [--far-plane FAR_PLANE] [--view-of-agents field of perception number of bins] [--view-of-walls field of perception number of bins]
+                            [--view-of-walls-matches]
+                            [trackset_file]
+
+View RoboFish tracks in a GUI.
+
+positional arguments:
+  trackset_file         Path to HDF5 file containing the tracks to view
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --draw-labels         Whether to draw labels inside the agents' outlines
+  --draw-view-vectors   Whether to draw view vectors to the right of / below the trackfile
+  --far-plane FAR_PLANE
+                        Maximum distance an agent can see
+  --view-of-agents field of perception number of bins
+  --view-of-walls field of perception number of bins
+  --view-of-walls-matches
+```
+
+</small>
\ No newline at end of file

docs/file.md

+`robofish.io.file.File` objects are the root of the project. The object contains all information about the environment, entities, and time.
+
+In the simplest form we define a new File with a world size in cm and a frequency in hz. Afterwards, we can save it with a path.
+
+```python
+import robofish.io
+
+# Create a new robofish io file
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+f.save_as("test.hdf5")
+```
+
+The File object can also be generated, with a given path. In this case, we work on the file directly. The `with` block ensures, that the file is validated after the block.
+
+```python
+with robofish.io.File(
+    "test.hdf5", mode="x", world_size_cm=[100, 100], frequency_hz=25.0
+) as f:
+    # Use file f here
+```
+
+When opening a file with a path, a mode should be specified to describe how the file should be opened.
+
+| Mode 	| Description                            	|
+|------	|----------------------------------------	|
+| r    	| Readonly, file must exist (default)    	|
+| r+   	| Read/write, file must exist            	|
+| w    	| Create file, truncate if exists        	|
+| x    	| Create file, fail if exists            	|
+| a    	| Read/write if exists, create otherwise 	|
+
+## Attributes
+
+Attributes of the file can be added, to describe the contents. 
+The attributes can be set like this:
+```python
+f.attrs["experiment_setup"] = "This file comes from the tutorial."
+f.attrs["experiment_issues"] = "All data in this file is made up."
+```
+
+Any attribute is allowed, but some cannonical attributes are prepared:<br>
+`publication_url, video_url, tracking_software_name, tracking_software_version, tracking_software_url, experiment_setup, experiment_issues`
+
+## Properties
+As described in `robofish.io`, all properties of `robofish.io.entity`s can be accessed by adding the prefix `entity_` to the function.
\ No newline at end of file

docs/index.md

+This package provides you:
+
+- Creation, storing, loading, modifying, inspecting of io-files.
+- Preprocessing (orientation calculation, action calculation, raycasting, ...)
+- Quick and easy evaluation of behavior
+- Data, which is interchangable between labs and tools. No conversions required, since units and coordinate systems are standardized.
+- No custom data import, just `include robofish.io`
+- This package is tested extensively
+
+Features coming up:
+
+- Interface for unified behavior models
+- Pytorch Datasets directly from `robofish.io` files.
+
+
+<a href="https://git.imp.fu-berlin.de/bioroboticslab/robofish/io" class="myButton">Code</a>
+
+## Installation
+Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) to your pip config and install the packagage.
+
+```bash
+python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple
+python3 -m pip install robofish-io robofish-trackviewer
+```
+
+## Usage
+
+This documentation is structured with increasing complexity.
+First we'll execute the example from the readme. More examples can be found in ```examples/```.
+
+```python
+# Create a new robofish io file
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+f.attrs["experiment_setup"] = "This is a simple example with made up data."
+
+# Create a new robot entity with 10 timesteps.
+# Positions and orientations are passed separately in this example.
+# Since the orientations have two columns, unit vectors are assumed
+# (orientation_x, orientation_y)
+f.create_entity(
+    category="robot",
+    name="robot",
+    positions=np.zeros((10, 2)),
+    orientations=np.ones((10, 2)) * [0, 1],
+)
+
+# Create a new fish entity with 10 timesteps.
+# In this case, we pass positions and orientations together (x, y, rad).
+# Since it is a 3 column array, orientations in radiants are assumed.
+poses = np.zeros((10, 3))
+poses[:, 0] = np.arange(-5, 5)
+poses[:, 1] = np.arange(-5, 5)
+poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
+fish = f.create_entity("fish", poses=poses)
+fish.attrs["species"] = "My rotating spaghetti fish"
+fish.attrs["fish_standard_length_cm"] = 10
+
+# Some possibilities to access the data
+print(f"The file:\n{f}")
+print(
+    f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+    + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+)
+print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
+
+# Save the file
+f.save_as("example.hdf5")
+```
+
+⚠️ Please try out the example on your computer and read the output.
+
+We created an `robofish.io.file.File` object, then we added two `robofish.io.entity.Entity` objects.
+Afterwards we read some properties of the file and printed them *(more info in [Reading Properties](#reading-properties))*.
+Lastly, we saved the file to `example.hdf5`.
+Congrats, you created your first io file. We'll continue working with it.
+
+---
+
+We can examine the file now by using commandline tools. These are some examples, more details in [Commandline Tools](## Commandline Tools)
+
+```bash
+robofish-io-print example.hdf5
+```
+Checking out the file content.
+
+```bash
+robofish-io-evaluate speed example.hdf5
+```
+Show a histogram of speeds in the file. For more evaluation options check `robofish-io-evaluate --help`
+
+```bash
+robofish-trackviewer example.hdf5
+```
+View a video of the track in an interactive window.
+
+Further details about the commandline tools can be found in `robofish.io.app`.
+
+## Accessing real data
+Until now, we only worked with dummy data. Data from different sources is available in the Trackdb. It is currently stored at the FU Box.
+
+⚠️ If you don't have access to the Trackdb yet, please text Andi by Mail or Mattermost (andi.gerken@gmail.com)
+
+
+## Reading properties
+
+Files and entities have usefull properties to access their content. In this way, positions, orientations, speeds, and turns can be accessed easily.
+
+All shown property functions can be called from a file or on one entity.
+The function names are identical but have a `entity_` prefix.
+
+```python
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+nemo = f.create_entity(category='fish', name='nemo', poses=np.zeros((10,3)))
+dori = f.create_entity(category='fish', name='dori', poses=np.zeros((10,3)))
+
+
+# Get the poses of nemo. Resulting in a (10,3) array
+print(nemo.poses_rad)
+
+# Get the poses of all entities. Resulting in a (2,10,3) array.
+print(f.entity_poses_rad)
+```
+
+In the same scheme the following properties are available:
+
+<small>
+
+| File/ Entity function       | Description                                                                                             |
+| --------------------------- | ------------------------------------------------------------------------------------------------------- |
+| *entity_*positions          | Positions as a `(*entities*, timesteps, 2 (x, y))` arary.                                               |
+| *entity_*orientations       | Orientations as a `(*entities*, timesteps, 2 (ori_x, ori_y))` arary.                                    |
+| *entity_*orientations_rad   | Orientations as a `(*entities*, timesteps, 1 (ori_rad))` arary.                                         |
+| *entity_*poses              | Poses as a `(*entities*, timesteps, 4 (x, y, x_ori, y_ori))` array.                                     |
+| *entity_*poses_rad          | Poses as a `(*entities*, timesteps, 3(x, y, ori_rad))` array.                                           |
+| *entity_*poses_calc_ori_rad | Poses with calculated orientations as a<br>`(*entities*, timesteps - 1, 3 (x, y, calc_ori_rad))` array. |
+| *entity_*speed_turn         | Speed and turn as a `(*entities*, timesteps - 2, 2 (speed_cm/s, turn_rad/s))` array.                    |
+
+</small>
+
+All these functions are described in more detail in `robofish.io.entity`.
+
+## Where to continue?
+We recommend continuing to read advanced options for `robofish.io.file`s and `robofish.io.entity`s.
+Create some files, validate them, look at them in the trackviewer, evaluate them.
+
+If you find bugs or get stuck somewhere, please text `Andi` on Mattermost or by mail ([andi.gerken@gmail.com](mailto:andi.gerken@gmail.com))
+
+
+---
+
+## Extended functions
+In the [Track Format Specification](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/uploads/f76d86e7a629ca38f472b8f23234dbb4/RoboFish_Track_Format_-_1.0.pdf) and in this package there are more possibilities, we don't describe explicitly here. If you want to use any of the following features, please ask Andi:
+
+Built in robofish.io and not described:
+
+- Multiple sampling frequencies
+- Timestamps per timestep
+- Calendar points
+- Outlines
+- Obstacles
+
+Planned in Track Format but not implemented:
+
+- 3D Tracks
+
+
+<style>
+.myButton {
+	background:#3971cc;
+	background-color:#3971cc;
+	border-radius:9px;
+	display:inline-block;
+	cursor:pointer;
+	color:#ffffff;
+	font-family:Arial;
+	font-size:20px;
+	padding:8px 16px;
+	text-decoration:none;
+}
+.myButton:hover {
+	background:#396bd1;
+	background-color:#396bd1;
+  color: #e0e0e0;
+  text-decoration: none;
+}
+.myButton:active {
+	position:relative;
+	top:1px;
+}
+</style>

examples/example_readme.py

@@ -7,32 +7,38 @@ def create_example_file(path):
     f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
     f.attrs["experiment_setup"] = "This is a simple example with made up data."
 
-    # Create a new robot entity. Positions and orientations are passed
-    # separately in this example. Since the orientations have two columns,
-    # unit vectors are assumed (orientation_x, orientation_y)
-    circle_rad = np.linspace(0, 2 * np.pi, num=100)
+    # Create a new robot entity with 10 timesteps.
+    # Positions and orientations are passed separately in this example.
+    # Since the orientations have two columns, unit vectors are assumed
+    # (orientation_x, orientation_y)
     f.create_entity(
         category="robot",
         name="robot",
-        positions=np.stack((np.cos(circle_rad), np.sin(circle_rad))).T * 40,
-        orientations=np.stack((-np.sin(circle_rad), np.cos(circle_rad))).T,
+        positions=np.zeros((10, 2)),
+        orientations=np.ones((10, 2)) * [0, 1],
     )
 
-    # Create a new fish entity.
+    # Create a new fish entity with 10 timesteps.
     # In this case, we pass positions and orientations together (x, y, rad).
     # Since it is a 3 column array, orientations in radiants are assumed.
-    poses = np.zeros((100, 3))
-    poses[:, 0] = np.arange(-50, 50)
-    poses[:, 1] = np.arange(-50, 50)
-    poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100)
+    poses = np.zeros((10, 3))
+    poses[:, 0] = np.arange(-5, 5)
+    poses[:, 1] = np.arange(-5, 5)
+    poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
     fish = f.create_entity("fish", poses=poses)
     fish.attrs["species"] = "My rotating spaghetti fish"
     fish.attrs["fish_standard_length_cm"] = 10
 
-    # Show and save the file
-    print(f)
-    print("Poses Shape: ", f.entity_poses.shape)
+    # Some possibilities to access the data
+    print(f"The file:\n{f}")
+    print(
+        f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+        + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+    )
+    print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+    print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
 
+    # Save the file
     f.save_as(path)
 
 

setup.py

@@ -48,7 +48,7 @@ setup(
     version=source_version(),
     author="",
     author_email="",
-    install_requires=["h5py>=2.10.0", "numpy", "seaborn", "pandas", "deprecation"],
+    install_requires=["h5py>=3.1.0", "numpy", "seaborn", "pandas", "deprecation"],
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Science/Research",

src/robofish/evaluate/app.py

@@ -11,6 +11,7 @@ Functions available to be used in the commandline to evaluate robofish.io files.
 
 import robofish.evaluate
 import argparse
+from pathlib import Path
 
 
 def function_dict():
@@ -25,6 +26,7 @@ def function_dict():
         "trajectories": base.evaluate_trajectories,
         "evaluate_positionVec": base.evaluate_positionVec,
         "follow_iid": base.evaluate_follow_iid,
+        "all": base.evaluate_all,
     }
 
 
@@ -41,24 +43,26 @@ def evaluate(args=None):
 
     fdict = function_dict()
 
+    longest_name = max([len(k) for k in fdict.keys()])
+
     parser = argparse.ArgumentParser(
-        description="This function can be called from the commandline to evaluate files.\
-        Different evaluation methods can be called, which generate graphs from the given files. \
-        \
-        With the first argument 'analysis_type', the type of analysis is chosen."
+        description="This function can be called from the commandline to evaluate files.\n"
+        + "Different evaluation methods can be called, which generate graphs from the given files.\n"
+        + "With the first argument 'analysis_type', the type of analysis is chosen.",
+        formatter_class=argparse.RawTextHelpFormatter,
     )
 
     parser.add_argument(
         "analysis_type",
         type=str,
         choices=fdict.keys(),
-        help="The type of analysis.\
-        speed - A histogram of speeds\
-        turn - A histogram of angular velocities\
-        tank_positions - A heatmap of the positions in the tank\
-        trajectories - A plot of all the trajectories\
-        follow_iid - A plot of the follow metric in relation to iid (inter individual distance)\
-        ",
+        help="The type of analysis.\n"
+        + "\n".join(
+            [
+                f"{key}{' ' * (longest_name - len(key))} - {func.__doc__.splitlines()[0]}"
+                for key, func in fdict.items()
+            ]
+        ),
     )
     parser.add_argument(
         "paths",
@@ -86,6 +90,11 @@ def evaluate(args=None):
         args = parser.parse_args()
 
     if args.analysis_type in fdict:
-        fdict[args.analysis_type](args.paths, args.names, args.save_path)
+        params = (args.paths, args.names, Path(args.save_path))
+        if args.analysis_type == "all":
+            normal_functions = function_dict()
+            normal_functions.pop("all")
+            params += (normal_functions,)
+        fdict[args.analysis_type](*params)
     else:
         print(f"Evaluation function not found {args.analysis_type}")

src/robofish/evaluate/evaluate.py

@@ -17,6 +17,7 @@ import numpy as np
 import pandas as pd
 from typing import Iterable
 from scipy import stats
+from tqdm import tqdm
 
 
 def get_all_poses_from_paths(paths: Iterable[str]):
@@ -123,6 +124,7 @@ def evaluate_turn(
     for k, files in enumerate(files_per_path):
         path_turns = []
         for p, file in files.items():
+            # TODO: Use new io functions (entity_poses_calc_ori_rad)
             poses = file.select_entity_poses(
                 None if predicate is None else predicate[k]
             )
@@ -676,7 +678,8 @@ def evaluate_follow_iid(
 def evaluate_all(
     paths: Iterable[str],
     labels: Iterable[str] = None,
-    save_folder: str = None,
+    save_folder: Path = None,
+    fdict: dict = None,
     predicate=None,
 ):
     """Generate all evaluation graphs and save them to a folder.
@@ -686,23 +689,15 @@ def evaluate_all(
         labels: Labels for the paths. If no labels are given, the paths will
             be used
         save_path: A path to a save location.
+        fdict: A dictionary of strings and evaluation functions
         predicate: a lambda function, selecting entities
             (example: lambda e: e.category == "fish")
     """
-    # save_folder = Path(save_folder)
-    evaluate_speed(paths, labels, save_folder + "speed.png", predicate)
-    evaluate_turn(paths, labels, save_folder + "turn.png", predicate)
-    evaluate_orientation(paths, labels, save_folder + "orientation.png", predicate)
-    evaluate_relativeOrientation(
-        paths, labels, save_folder + "relativeOrientation.png", predicate
-    )
-    evaluate_distanceToWall(
-        paths, labels, save_folder + "distanceToWall.png", predicate
-    )
-    evaluate_tankpositions(paths, labels, save_folder + "tankpositions.png", predicate)
-    evaluate_trajectories(paths, labels, save_folder + "trajectories.png", predicate)
-    evaluate_positionVec(paths, labels, save_folder + "posVec.png", predicate)
-    evaluate_follow_iid(paths, labels, save_folder + "follow_iid.png", predicate)
+    t = tqdm(fdict.items(), desc="Evaluation", leave=True)
+    for f_name, f_callable in t:
+        t.set_description(f_name)
+        t.refresh()  # to show immediately the update
+        f_callable(paths, labels, save_folder / (f_name + ".png"), predicate)
 
 
 def calculate_follow(a, b):

src/robofish/io/__init__.py

 # SPDX-License-Identifier: LGPL-3.0-or-later
 
+"""
+The Python package <a href="https://git.imp.fu-berlin.de/bioroboticslab/robofish/io">robofish.io</a> provides a simple interface to create, load, modify, and inspect files containing world information and movement tracks of entities (organisms, robots, obstacles,...).
+The files are saved in the `.hdf5` format and following the [Track Format Specification](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/uploads/f76d86e7a629ca38f472b8f23234dbb4/RoboFish_Track_Format_-_1.0.pdf).
+
+.. include:: ../../../docs/index.md
+"""
+
 import sys
 import logging
 

src/robofish/io/app.py

 # -*- coding: utf-8 -*-
 
+"""
+.. include:: ../../../docs/app.md
+"""
+
 # -----------------------------------------------------------
-# Functions available to be used in the commandline to show and validate hdf5
-# according to the Robofish track format (1.0 Draft 7). The standard is
-# available at https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format
-#
 # Dec 2020 Andreas Gerken, Berlin, Germany
 # Released under GNU 3.0 License
 # email andi.gerken@gmail.com

src/robofish/io/entity.py

+"""
+.. include:: ../../../docs/entity.md
+"""
+
 import robofish.io
 import robofish.io.utils as utils
 
@@ -134,31 +138,41 @@ class Entity(h5py.Group):
     def orientations(self):
         if not "orientations" in self:
             # If no orientation is given, the default direction is to the right
-            return np.tile([1, 0], (self.positions.shape[0], 1))
+            return np.tile([0, 1], (self.positions.shape[0], 1))
         return self["orientations"]
 
     @property
-    def orientations_calculated(self):
-        diff = np.diff(self.positions, axis=0)
-        angles = np.arctan2(diff[:, 1], diff[:, 0])
-        return angles[:, np.newaxis]
+    def orientations_rad(self):
+        ori_rad = utils.limit_angle_range(
+            np.arctan2(self.orientations[:, 1], self.orientations[:, 0]),
+            _range=(0, 2 * np.pi),
+        )
+        return ori_rad[:, np.newaxis]
 
     @property
     def poses_calc_ori_rad(self):
-        return np.concatenate(
-            [self.positions[:-1], self.orientations_calculated], axis=1
+        # Diff between positions [t - 1, 2]
+        diff = np.diff(self.positions, axis=0)
+
+        # angles [t - 1]
+        angles = utils.limit_angle_range(
+            np.arctan2(diff[:, 1], diff[:, 0]), _range=(0, 2 * np.pi)
+        )
+
+        # Positions with angles. The first position is cut of, as it does not have an orientation.
+        poses_with_calculated_orientation = np.concatenate(
+            [self.positions[1:], angles[:, np.newaxis]], axis=1
         )
 
+        return poses_with_calculated_orientation
+
     @property
     def poses(self):
         return np.concatenate([self.positions, self.orientations], axis=1)
 
     @property
     def poses_rad(self):
-        poses = self.poses
-        # calculate the angles from the orientation vectors, write them to the third row and delete the fourth row
-        ori_rad = np.arctan2(poses[:, 3], poses[:, 2])
-        return np.concatenate([poses[:, :2], ori_rad[:, np.newaxis]], axis=1)
+        return np.concatenate([self.positions, self.orientations_rad], axis=1)
 
     @property
     def speed_turn(self):
@@ -173,10 +187,12 @@ class Entity(h5py.Group):
             We assume, that the entity is oriented "correctly" in the first pose. So the first turn angle is 0.
         """
 
-        diff = np.diff(self.positions, axis=0)
-        speed = np.linalg.norm(diff, axis=1)
-        angles = np.arctan2(diff[:, 1], diff[:, 0])
-        turn = np.zeros_like(angles)
-        turn[0] = 0
-        turn[1:] = utils.limit_angle_range(np.diff(angles))
+        # poses with calulated orientation have first position cut of as it does not have an orientation
+        # (t - 1, (x ,y, ori))
+        poses_calc_ori = self.poses_calc_ori_rad
+
+        # Differences cuts of last item (t - 2, (dx, dy, d ori))
+        diff = np.diff(poses_calc_ori, axis=0)
+        speed = np.linalg.norm(diff[:, :2], axis=1)
+        turn = utils.limit_angle_range(diff[:, 2], _range=(-np.pi, np.pi))
         return np.stack([speed, turn], axis=-1)

src/robofish/io/file.py

 # -*- coding: utf-8 -*-
 
+"""
+.. include:: ../../../docs/file.md
+"""
+
 # -----------------------------------------------------------
 # Utils functions for reading, validating and writing hdf5 files according to
 # Robofish track format (1.0 Draft 7). The standard is available at
 # https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format
-#
-# The term track is used to describe a dictionary, describing the track in a dict.
-# To distinguish between attributes, dictionaries and groups, a prefix is used
-# (a_ for attribute, d_ for dictionary, and g_ for groups).
+
 #
 # Dec 2020 Andreas Gerken, Berlin, Germany
 # Released under GNU 3.0 License
@@ -30,6 +31,7 @@ import uuid
 import deprecation
 import types
 
+# Remember: Update docstring when updating these two global variables
 default_format_version = np.array([1, 0], dtype=np.int32)
 
 default_format_url = (
@@ -61,26 +63,64 @@ class File(h5py.File):
     ):
         """Create a new RoboFish Track Format object.
 
-        When called with a path, it is loaded, otherwise a new temporary file is created.
+        When called with a path, it is loaded, otherwise a new temporary
+        file is created. File contents can be validated against the
+        track format specification.
 
         Parameters
         ----------
         path : str or Path, optional
             Location of file to be opened. If not provided, mode is ignored.
-
-        mode : str
-            r        Readonly, file must exist (default)
-            r+       Read/write, file must exist
-            w        Create file, truncate if exists
-            x        Create file, fail if exists
-            a        Read/write if exists, create otherwise
-
-        world_size_cm
-            optional integer array of the world size in cm
-        strict_validate
-            optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
-        format_version
-            optional version [major, minor] of the trackformat specification
+        mode : str, default='r'
+            'r'        Readonly, file must exist
+            'r+'       Read/write, file must exist
+            'w'        Create file, truncate if exists
+            'x'        Create file, fail if exists
+            'a'        Read/write if exists, create otherwise
+        world_size_cm : [int, int] , optional
+            side lengths [x, y] of the world in cm.
+            rectangular world shape is assumed.
+            TODO: Cuboid world is also possible in track format
+        strict_validate : bool, default=False
+            if the file should be strictly validated against the track
+            format specification, when loaded from a path.
+            TODO: Should this validate against the version sepcified in
+            format_version or just against the most recent version?
+        format_version : [int, int], default=[1,0]
+            version [major, minor] of track format specification
+        format_url : str, default="https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/-/releases/1.0"
+            location of track format specification.
+            should fit `format_version`.
+        sampling_name : str, optional
+            How to specify your sampling:
+
+            1. (optional)
+                provide text description of your sampling in `sampling_name`
+
+            2.a (mandatory, if you have a constant sampling frequency)
+                specify `frequency_hz` with your sampling frequency in Hz
+
+            2.b (mandatory, if you do NOT have a constant sampling frequency)
+                specify `monotonic_time_points_us` with a list[1] of time
+                points in microseconds on a montonic clock, one for each
+                sample in your dataset.
+
+            3.  (optional)
+                specify `calendar_time_points` with a list[2] of time points
+                in the ISO 8601 extended format with microsecond precision
+                and time zone designator[3],  one for each sample in your
+                dataset.
+
+            [1] any Iterable of int
+            [2] any Iterable of str
+            [3] example:  "2020-11-18T13:21:34.117015+01:00"
+
+        frequency_hz: int, optional
+            refer to explanation of `sampling_name`
+        monotonic_time_points_us: Iterable of int, optional
+            refer to explanation of `sampling_name`
+        calendar_time_points: Iterable of str, optional
+            refer to explanation of `sampling_name`
         """
 
         if path is None:
@@ -143,6 +183,8 @@ class File(h5py.File):
         Args:
             path: path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
             strict_validate: optional boolean, if the file should be strictly validated, before saving. The default is True.
+        Returns:
+            The file itself, so something like f = robofish.io.File().save_as("file.hdf5") works
         """
 
         self.validate(strict_validate=strict_validate)
@@ -155,6 +197,8 @@ class File(h5py.File):
 
         shutil.copyfile(Path(self.filename).resolve(), path)
 
+        return self
+
     def create_sampling(
         self,
         name: str = None,
@@ -190,10 +234,10 @@ class File(h5py.File):
             def format_calendar_time_point(p):
                 if isinstance(p, datetime.datetime):
                     assert p.tzinfo is not None, "Missing timezone for calendar point."
-                    return p.isoformat(timespec="milliseconds")
+                    return p.isoformat(timespec="microseconds")
                 elif isinstance(p, str):
                     assert p == datetime.datetime.fromisoformat(p).isoformat(
-                        timespec="milliseconds"
+                        timespec="microseconds"
                     )
                     return p
                 else:
@@ -205,8 +249,6 @@ class File(h5py.File):
                 format_calendar_time_point(p) for p in calendar_time_points
             ]
 
-            for c in calendar_time_points:
-                print(type(c))
             sampling.create_dataset(
                 "calendar_time_points",
                 data=calendar_time_points,
@@ -271,7 +313,7 @@ class File(h5py.File):
 
         Args:
             TODO
-            category: the  of the entity. The canonical values are ['fish', 'robot', 'obstacle'].
+            category: the  of the entity. The canonical values are ['organism', 'robot', 'obstacle'].
             poses: optional two dimensional array, containing the poses of the entity (x,y,orientation_x, orientation_y).
             poses_rad: optional two dimensional containing the poses of the entity (x,y, orientation_rad).
             name: optional name of the entity. If no name is given, the  is used with an id (e.g. 'fish_1')
@@ -309,7 +351,7 @@ class File(h5py.File):
         """Creates multiple entities.
 
         Args:
-            category: The common category for the entities. The canonical values are ['fish', 'robot', 'obstacle'].
+            category: The common category for the entities. The canonical values are ['organism', 'robot', 'obstacle'].
             poses: three dimensional array, containing the poses of the entity.
             name: optional array of names of the entities. If no names are given, the category is used with an id (e.g. 'fish_1')
             outlines: optional array, containing the outlines of the entities, either a three dimensional common outline array can be given, or a four dimensional array.
@@ -321,7 +363,6 @@ class File(h5py.File):
         assert poses.ndim == 3
         assert poses.shape[2] in [3, 4]
         agents = poses.shape[0]
-        timesteps = poses.shape[1]
         entity_names = []
 
         for i in range(agents):
@@ -357,9 +398,23 @@ class File(h5py.File):
             for name in self.entity_names
         ]
 
+    @property
+    def entity_positions(self):
+        return self.select_entity_property(None, entity_property=Entity.positions)
+
+    @property
+    def entity_orientations(self):
+        return self.select_entity_property(None, entity_property=Entity.orientations)
+
+    @property
+    def entity_orientations_rad(self):
+        return self.select_entity_property(
+            None, entity_property=Entity.orientations_rad
+        )
+
     @property
     def entity_poses(self):
-        return self.select_entity_property(None)
+        return self.select_entity_property(None, entity_property=Entity.poses)
 
     @property
     def entity_poses_rad(self):

src/robofish/io/io.py

@@ -16,7 +16,7 @@ def now_iso8061() -> str:
         str: The current time as iso8061 string.
     """
     return datetime.datetime.now(datetime.timezone.utc).isoformat(
-        timespec="milliseconds"
+        timespec="microseconds"
     )
 
 

src/robofish/io/validation.py

@@ -113,9 +113,7 @@ def validate(iofile: File, strict_validate: bool = True) -> (bool, str):
                 if a in sampling:
                     assert_validate_type(sampling[a], a_type, a, f"sampling {s_name}")
 
-            if "frequency_hz" in sampling.attrs:
-                pass
-            elif "monotonic_time_points_us" in sampling:
+            if "monotonic_time_points_us" in sampling:
                 time_points = sampling["monotonic_time_points_us"]
                 # 1 dimensional array
                 assert_validate(
@@ -138,6 +136,7 @@ def validate(iofile: File, strict_validate: bool = True) -> (bool, str):
                     "Dimensionality of calendar_time_points should be 1",
                     f"sampling {s_name}",
                 )
+                if "monotonic_time_points_us" in sampling:
                     assert_validate(
                         calendar_points.shape[0] == time_points.shape[0],
                         "The length of calendar points (%d) does not match the length of monotonic points (%d)"

tests/robofish/evaluate/test_app_evaluate.py

@@ -23,5 +23,6 @@ def test_app_validate():
             self.save_path = graphics_out
 
     for mode in app.function_dict().keys():
+        if mode != "all":
             app.evaluate(DummyArgs(mode))
     graphics_out.unlink()

tests/robofish/io/test_entity.py

@@ -6,9 +6,9 @@ import numpy as np
 def test_entity_object():
     sf = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25)
     f = sf.create_entity("fish", positions=[[10, 10]])
-    assert type(f) == robofish.io.Entity
-    assert f.name == "fish_1"
-    assert f.attrs["category"] == "fish"
+    assert type(f) == robofish.io.Entity, "Type of entity was wrong"
+    assert f.name == "fish_1", "Name of entity was wrong"
+    assert f.attrs["category"] == "fish", "category was wrong"
     print(dir(f))
     print(f["positions"])
     assert type(f["positions"]) == h5py.Dataset
@@ -40,13 +40,13 @@ def test_entity_turn_speed():
     )
     e = f.create_entity("fish", positions=positions)
     speed_turn = e.speed_turn
-    assert speed_turn.shape == (99, 2)
+    assert speed_turn.shape == (98, 2)
 
-    # No turn in the first timestep, since initialization turns it the right way
-    assert speed_turn[0, 1] == 0
+    # Turns and speeds shoud be all the same afterwards, since the fish swims with constant velocity and angular velocity.
+    assert (np.std(speed_turn, axis=0) < 0.0001).all()
 
-    # Turns and speeds shoud afterwards be all the same afterwards, since the fish swims with constant velocity and angular velocity.
-    assert (np.std(speed_turn[1:], axis=0) < 0.0001).all()
+    # Cut off the first position as it cannot be simulated
+    positions = positions[1:]
 
     # Use turn_speed to generate positions
     gen_positions = np.zeros((positions.shape[0], 3))

tests/robofish/io/test_file.py

@@ -112,7 +112,7 @@ def test_multiple_entities():
     # create new sampling
     m_points = np.ones((timesteps))
     c_points = np.empty((timesteps), dtype="O")
-    c_points[:5] = "2020-12-02T10:21:58.100+00:00"
+    c_points[:5] = "2020-12-02T10:21:58.100000+00:00"
     c_points[5:] = robofish.io.now_iso8061()
 
     new_sampling = sf.create_sampling(
@@ -184,8 +184,10 @@ def test_entity_positions_no_orientation():
         # Create an entity, using radiants
         f.create_entity("fish", positions=np.ones((100, 2)))
 
+        # In poses, the default orientation pointing up should be added.
         assert f.entity_poses.shape == (1, 100, 4)
-        assert (f.entity_poses[:, :] == np.array([1, 1, 1, 0])).all()
+        assert (f.entity_poses[:, :] == np.array([1, 1, 0, 1])).all()
+        assert np.isclose(f.entity_orientations_rad, np.pi / 2).all()
 
         # Calculate the orientation
         assert f.entity_poses_calc_ori_rad.shape == (1, 99, 3)