diff --git a/.gitignore b/.gitignore
index a6e090246416a5e7f96de1964f942fabf9776d9a..3825a6ab5a89436ea58a3513859272e88abee742 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,7 +9,6 @@ dist
report.xml
htmlcov
html
-docs
env
!tests/resources/*.hdf5
diff --git a/README.md b/README.md
index 25c87abf39fdc4e42132ba0e9d7565209c6b26cd..8e6eafc69af1da3141a0e49008a81771e199e4ca 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
[](https://git.imp.fu-berlin.de/bioroboticslab/robofish/io/commits/master)
# Robofish IO
-This repository implements an easy to use interface, to create, save, load, and work [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files.
+This repository implements an easy to use interface, to create, save, load, and work with [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files.
## <a href="http://agerken.de/io/index.html">Documentation</a>
@@ -19,52 +19,50 @@ Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robof
```bash
python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple
-python3 -m pip install robofish-io robofish-trackviewer
+python3 -m pip install robofish-io
```
-Better variant:
-- Follow instructions at [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts)
-- ```pip3 install robofish-io```
-
-
## Usage
We show a simple example below. More examples can be found in ```examples/```
```python
-import robofish.io
-import numpy as np
-
-
# Create a new robofish io file
f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
f.attrs["experiment_setup"] = "This is a simple example with made up data."
-# Create a new robot entity. Positions and orientations are passed
-# separately in this example. Since the orientations have two columns,
-# unit vectors are assumed (orientation_x, orientation_y)
+# Create a new robot entity with 10 timesteps.
+# Positions and orientations are passed separately in this example.
+# Since the orientations have two columns, unit vectors are assumed
+# (orientation_x, orientation_y)
f.create_entity(
category="robot",
name="robot",
- positions=np.zeros((100, 2)),
- orientations=np.ones((100, 2)) * [0, 1],
+ positions=np.zeros((10, 2)),
+ orientations=np.ones((10, 2)) * [0, 1],
)
-# Create a new fish entity.
+# Create a new fish entity with 10 timesteps.
# In this case, we pass positions and orientations together (x, y, rad).
# Since it is a 3 column array, orientations in radiants are assumed.
-poses = np.zeros((100, 3))
-poses[:, 0] = np.arange(-50, 50)
-poses[:, 1] = np.arange(-50, 50)
-poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100)
+poses = np.zeros((10, 3))
+poses[:, 0] = np.arange(-5, 5)
+poses[:, 1] = np.arange(-5, 5)
+poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
fish = f.create_entity("fish", poses=poses)
fish.attrs["species"] = "My rotating spaghetti fish"
fish.attrs["fish_standard_length_cm"] = 10
-# Show and save the file
-print(f)
-print("Poses Shape: ", f.entity_poses.shape)
+# Some possibilities to access the data
+print(f"The file:\n{f}")
+print(
+ f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+ + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+)
+print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
-f.save_as(path)
+# Save the file
+f.save_as("example.hdf5")
```
### Evaluation
@@ -74,14 +72,4 @@ Current modes are:
- turn
- tank_positions
- trajectories
-- follow_iid
-
-
-## LICENSE
-
-This work is licensed under LGPL 3.0 (or any later version).
-Individual files contain the following tag instead of the full license text:
-
-`SPDX-License-Identifier: LGPL-3.0-or-later`
-
-This enables machine processing of license information based on the SPDX License Identifiers available here: https://spdx.org/licenses/
+- follow_iid
\ No newline at end of file
diff --git a/docs/app.md b/docs/app.md
new file mode 100644
index 0000000000000000000000000000000000000000..54bb225a100dd4f7311d1e7fa4abb5b7dddf6248
--- /dev/null
+++ b/docs/app.md
@@ -0,0 +1,99 @@
+This module defines the command line interface.
+All commands have a ``--help`` option to get more info about them in the command line.
+
+
+## Print
+```bash
+robofish-io-print example.hdf5
+```
+Checking out the content of files.
+
+<small>
+
+```
+usage: robofish-io-print [-h] [--output_format {shape,full}] path
+
+This function can be used to print hdf5 files from the command line
+positional arguments:
+ path The path to a hdf5 file
+optional arguments:
+ -h, --help show this help message and exit
+ --output_format {shape,full}
+ Choose how datasets are printed, either the shapes or the full content is printed
+```
+
+</small>
+
+## Evaluate
+```bash
+robofish-io-evaluate *analysis_type* example.hdf5
+```
+Show some property of a file.
+
+
+<small>
+
+```
+usage: robofish-io-evaluate [-h] [--names NAMES [NAMES ...]] [--save_path SAVE_PATH]
+ {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid} paths [paths ...]
+
+This function can be called from the commandline to evaluate files.
+Different evaluation methods can be called, which generate graphs from the given files.
+With the first argument 'analysis_type', the type of analysis is chosen.
+
+positional arguments:
+ {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid}
+ The type of analysis.
+ speed - Evaluate the speed of the entities as histogram.
+ turn - Evaluate the turn angles of the entities as histogram.
+ orientation - Evaluate the orientations of the entities on a 2d grid.
+ relative_orientation - Evaluate the relative orientations of the entities as a histogram.
+ distance_to_wall - Evaluate the distances of the entities to the walls as a histogram.
+ tank_positions - Evaluate the positions of the entities as a heatmap.
+ trajectories - Evaluate the trajectories of the entities.
+ evaluate_positionVec - Evaluate the vectors pointing from the focal fish to the conspecifics as heatmap.
+ follow_iid - Evaluate the follow metric in respect to the inter individual distance (iid).
+ paths The paths to files or folders. Multiple paths can be given to compare experiments.
+
+optional arguments:
+ -h, --help show this help message and exit
+ --names NAMES [NAMES ...]
+ Names, that should be used in the graphs instead of the pahts.
+ --save_path SAVE_PATH
+ Filename for saving resulting graphics.
+```
+
+</small>
+
+## Trackviewer
+
+```bash
+robofish-trackviewer example.hdf5
+```
+
+The trackviewer is from a different repository. It was included in the install instructions.
+
+<small>
+
+```
+usage: robofish-trackviewer [-h] [--draw-labels] [--draw-view-vectors] [--far-plane FAR_PLANE] [--view-of-agents field of perception number of bins] [--view-of-walls field of perception number of bins]
+ [--view-of-walls-matches]
+ [trackset_file]
+
+View RoboFish tracks in a GUI.
+
+positional arguments:
+ trackset_file Path to HDF5 file containing the tracks to view
+
+optional arguments:
+ -h, --help show this help message and exit
+ --draw-labels Whether to draw labels inside the agents' outlines
+ --draw-view-vectors Whether to draw view vectors to the right of / below the trackfile
+ --far-plane FAR_PLANE
+ Maximum distance an agent can see
+ --view-of-agents field of perception number of bins
+ --view-of-walls field of perception number of bins
+ --view-of-walls-matches
+```
+
+</small>
\ No newline at end of file
diff --git a/docs/entity.md b/docs/entity.md
new file mode 100644
index 0000000000000000000000000000000000000000..58b5f8606ae9992adf49efb74b8f3363ea5f9c1b
--- /dev/null
+++ b/docs/entity.md
@@ -0,0 +1,88 @@
+Tracks of entities are stored in `robofish.io.entity.Entity` objects. They are created by the `robofish.io.file.File` object. They require a category and can have a name. If no name is given, it will be created, using the category and an id.
+
+```python
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+nemo = f.create_entity(category="fish", name="nemo")
+```
+
+The function `create_entity()` returns an entity object. It can be stored but it's not neccessary. The entity is automatically stored in the file.
+
+## Pose options
+
+The poses of an entity can be passed in multiple ways. Poses are divided into `positions` (x, y) and `orientations`.
+Orientations are internally represented in unit vectors but can also be passed in rads.
+The shape of the passed orientation array defines the meaning.
+
+```python
+# Create dummy poses for 100 timesteps
+pos = np.zeros((100, 2))
+ori_vec = np.zeros((100,2)) * [0, 1]
+ori_rad = np.zeros((100,1))
+
+# Creating an entity without orientations. Here we keep the entity object, to use it later.
+f.create_entity(category="fish", positions=pos)
+# Creating an entity using orientation vectors. Keeping the entity object is not neccessary, it is saved in the file.
+f.create_entity(category="fish", positions=pos, orientations=ori_vec)
+# Creating an entity using radiant orientations.
+f.create_entity(category="fish", positions=pos, orientations=ori_rad)
+```
+
+The poses can be also passed in an combined array.
+```python
+# Create an entity using orientation vectors.
+f.create_entity(category="fish", poses=np.ones((100,4)) * np.sqrt(2))
+# Create an entity using radiant orientations
+f.create_entity(category="fish", poses=np.zeros((100,3)))
+```
+
+## Creating multiple entities at once
+
+Multiple entities can be created at once.
+```python
+# Here we create 4 fishes from poses with radiant orientation
+f.create_multiple_entities(category="fish", poses=np.zeros((4,100,3)))
+```
+
+## Attributes
+
+Entities can have attributes to describe them.
+
+The attributes can be set like this:
+```python
+nemo.attrs["species"] = "Clownfish"
+nemo.attrs["fish_standard_length_cm"] = 10
+```
+
+Any attribute is allowed, but some cannonical attributes are prepared:<br>
+`species`:`str`, `sex`:`str`, `fish_standard_length_cm`:`float`
+
+
+## Properties
+
+As described in `robofish.io`, Files and Entities have useful properties.
+
+<small>
+
+| Entity function | Description |
+| ---------------------------------------------- | ------------------------------------------------------------------------------------------- |
+| `robofish.io.entity.Entity.positions` | Positions as a `(timesteps, 2 (x, y))` arary. |
+| `robofish.io.entity.Entity.orientations` | Orientations as a `(timesteps, 2 (ori_x, ori_y))` arary. |
+| `robofish.io.entity.Entity.orientations_rad` | Orientations as a `(timesteps, 1 (ori_rad))` arary. |
+| `robofish.io.entity.Entity.poses` | Poses as a `(timesteps, 4 (x, y, x_ori, y_ori))` array. |
+| `robofish.io.entity.Entity.poses_rad` | Poses as a `(timesteps, 3(x, y, ori_rad))` array. |
+| `robofish.io.entity.Entity.poses_calc_ori_rad` | Poses with calculated orientations as a<br>`(timesteps - 1, 3 (x, y, calc_ori_rad))` array. |
+| `robofish.io.entity.Entity.speed_turn` | Speed and turn as a `(timesteps - 2, 2 (speed_cm/s, turn_rad/s))` array. |
+
+</small>
+
+### Calculated orientations and speeds.
+
+
+
+The image shows, how the orientations are calculated (`robofish.io.entity.Entity.poses_calc_ori_rad`), the orientations always show away from the last position. In this way, the first position does not have an orientation and the shape of the resulting array is `(timesteps - 1, 3 (x, y, calc_ori_rad))`.
+
+The meaning of the `robofish.io.entity.Entity.speed_turn` can also be explained with the image. The turn is the angle, the agent has to turn, to orientate towards the next position. The speed is the calculated, using the distance between two positions. The resulting turn and speed is converted to `[rad/s]` and `[cm/s]`. The first and last position don't have any action which results in an array shape of `(timesteps - 2, 2 (speed, turn))`.
+
+---
+
+⚠️ Try this out by extending the example of the main doc, so that a new teleporting fish with random positions [-50, 50] is generated. How does that change the output and speed histogram `robofish-io-evaluate speed example.hdf5`? Try writing some attributes.
diff --git a/docs/file.md b/docs/file.md
new file mode 100644
index 0000000000000000000000000000000000000000..bdfbfba3cca6d8201b8b8e0ec5983f5b8670dfab
--- /dev/null
+++ b/docs/file.md
@@ -0,0 +1,45 @@
+`robofish.io.file.File` objects are the root of the project. The object contains all information about the environment, entities, and time.
+
+In the simplest form we define a new File with a world size in cm and a frequency in hz. Afterwards, we can save it with a path.
+
+```python
+import robofish.io
+
+# Create a new robofish io file
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+f.save_as("test.hdf5")
+```
+
+The File object can also be generated, with a given path. In this case, we work on the file directly. The `with` block ensures, that the file is validated after the block.
+
+```python
+with robofish.io.File(
+ "test.hdf5", mode="x", world_size_cm=[100, 100], frequency_hz=25.0
+) as f:
+ # Use file f here
+```
+
+When opening a file with a path, a mode should be specified to describe how the file should be opened.
+
+| Mode | Description |
+|------ |---------------------------------------- |
+| r | Readonly, file must exist (default) |
+| r+ | Read/write, file must exist |
+| w | Create file, truncate if exists |
+| x | Create file, fail if exists |
+| a | Read/write if exists, create otherwise |
+
+## Attributes
+
+Attributes of the file can be added, to describe the contents.
+The attributes can be set like this:
+```python
+f.attrs["experiment_setup"] = "This file comes from the tutorial."
+f.attrs["experiment_issues"] = "All data in this file is made up."
+```
+
+Any attribute is allowed, but some cannonical attributes are prepared:<br>
+`publication_url, video_url, tracking_software_name, tracking_software_version, tracking_software_url, experiment_setup, experiment_issues`
+
+## Properties
+As described in `robofish.io`, all properties of `robofish.io.entity`s can be accessed by adding the prefix `entity_` to the function.
\ No newline at end of file
diff --git a/docs/img/calc_ori_speed_turn.png b/docs/img/calc_ori_speed_turn.png
new file mode 100644
index 0000000000000000000000000000000000000000..439d4c95455a2c2ee8f30ee616ccf2420277a5e3
Binary files /dev/null and b/docs/img/calc_ori_speed_turn.png differ
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 0000000000000000000000000000000000000000..df674a31b092d893994a20961bf90ebcebbaf5ea
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,191 @@
+This package provides you:
+
+- Creation, storing, loading, modifying, inspecting of io-files.
+- Preprocessing (orientation calculation, action calculation, raycasting, ...)
+- Quick and easy evaluation of behavior
+- Data, which is interchangable between labs and tools. No conversions required, since units and coordinate systems are standardized.
+- No custom data import, just `include robofish.io`
+- This package is tested extensively
+
+Features coming up:
+
+- Interface for unified behavior models
+- Pytorch Datasets directly from `robofish.io` files.
+
+
+<a href="https://git.imp.fu-berlin.de/bioroboticslab/robofish/io" class="myButton">Code</a>
+
+## Installation
+Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) to your pip config and install the packagage.
+
+```bash
+python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple
+python3 -m pip install robofish-io robofish-trackviewer
+```
+
+## Usage
+
+This documentation is structured with increasing complexity.
+First we'll execute the example from the readme. More examples can be found in ```examples/```.
+
+```python
+# Create a new robofish io file
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+f.attrs["experiment_setup"] = "This is a simple example with made up data."
+
+# Create a new robot entity with 10 timesteps.
+# Positions and orientations are passed separately in this example.
+# Since the orientations have two columns, unit vectors are assumed
+# (orientation_x, orientation_y)
+f.create_entity(
+ category="robot",
+ name="robot",
+ positions=np.zeros((10, 2)),
+ orientations=np.ones((10, 2)) * [0, 1],
+)
+
+# Create a new fish entity with 10 timesteps.
+# In this case, we pass positions and orientations together (x, y, rad).
+# Since it is a 3 column array, orientations in radiants are assumed.
+poses = np.zeros((10, 3))
+poses[:, 0] = np.arange(-5, 5)
+poses[:, 1] = np.arange(-5, 5)
+poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
+fish = f.create_entity("fish", poses=poses)
+fish.attrs["species"] = "My rotating spaghetti fish"
+fish.attrs["fish_standard_length_cm"] = 10
+
+# Some possibilities to access the data
+print(f"The file:\n{f}")
+print(
+ f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+ + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+)
+print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
+
+# Save the file
+f.save_as("example.hdf5")
+```
+
+⚠️ Please try out the example on your computer and read the output.
+
+We created an `robofish.io.file.File` object, then we added two `robofish.io.entity.Entity` objects.
+Afterwards we read some properties of the file and printed them *(more info in [Reading Properties](#reading-properties))*.
+Lastly, we saved the file to `example.hdf5`.
+Congrats, you created your first io file. We'll continue working with it.
+
+---
+
+We can examine the file now by using commandline tools. These are some examples, more details in [Commandline Tools](## Commandline Tools)
+
+```bash
+robofish-io-print example.hdf5
+```
+Checking out the file content.
+
+```bash
+robofish-io-evaluate speed example.hdf5
+```
+Show a histogram of speeds in the file. For more evaluation options check `robofish-io-evaluate --help`
+
+```bash
+robofish-trackviewer example.hdf5
+```
+View a video of the track in an interactive window.
+
+Further details about the commandline tools can be found in `robofish.io.app`.
+
+## Accessing real data
+Until now, we only worked with dummy data. Data from different sources is available in the Trackdb. It is currently stored at the FU Box.
+
+⚠️ If you don't have access to the Trackdb yet, please text Andi by Mail or Mattermost (andi.gerken@gmail.com)
+
+
+## Reading properties
+
+Files and entities have usefull properties to access their content. In this way, positions, orientations, speeds, and turns can be accessed easily.
+
+All shown property functions can be called from a file or on one entity.
+The function names are identical but have a `entity_` prefix.
+
+```python
+f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
+nemo = f.create_entity(category='fish', name='nemo', poses=np.zeros((10,3)))
+dori = f.create_entity(category='fish', name='dori', poses=np.zeros((10,3)))
+
+
+# Get the poses of nemo. Resulting in a (10,3) array
+print(nemo.poses_rad)
+
+# Get the poses of all entities. Resulting in a (2,10,3) array.
+print(f.entity_poses_rad)
+```
+
+In the same scheme the following properties are available:
+
+<small>
+
+| File/ Entity function | Description |
+| --------------------------- | ------------------------------------------------------------------------------------------------------- |
+| *entity_*positions | Positions as a `(*entities*, timesteps, 2 (x, y))` arary. |
+| *entity_*orientations | Orientations as a `(*entities*, timesteps, 2 (ori_x, ori_y))` arary. |
+| *entity_*orientations_rad | Orientations as a `(*entities*, timesteps, 1 (ori_rad))` arary. |
+| *entity_*poses | Poses as a `(*entities*, timesteps, 4 (x, y, x_ori, y_ori))` array. |
+| *entity_*poses_rad | Poses as a `(*entities*, timesteps, 3(x, y, ori_rad))` array. |
+| *entity_*poses_calc_ori_rad | Poses with calculated orientations as a<br>`(*entities*, timesteps - 1, 3 (x, y, calc_ori_rad))` array. |
+| *entity_*speed_turn | Speed and turn as a `(*entities*, timesteps - 2, 2 (speed_cm/s, turn_rad/s))` array. |
+
+</small>
+
+All these functions are described in more detail in `robofish.io.entity`.
+
+## Where to continue?
+We recommend continuing to read advanced options for `robofish.io.file`s and `robofish.io.entity`s.
+Create some files, validate them, look at them in the trackviewer, evaluate them.
+
+If you find bugs or get stuck somewhere, please text `Andi` on Mattermost or by mail ([andi.gerken@gmail.com](mailto:andi.gerken@gmail.com))
+
+
+---
+
+## Extended functions
+In the [Track Format Specification](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/uploads/f76d86e7a629ca38f472b8f23234dbb4/RoboFish_Track_Format_-_1.0.pdf) and in this package there are more possibilities, we don't describe explicitly here. If you want to use any of the following features, please ask Andi:
+
+Built in robofish.io and not described:
+
+- Multiple sampling frequencies
+- Timestamps per timestep
+- Calendar points
+- Outlines
+- Obstacles
+
+Planned in Track Format but not implemented:
+
+- 3D Tracks
+
+
+<style>
+.myButton {
+ background:#3971cc;
+ background-color:#3971cc;
+ border-radius:9px;
+ display:inline-block;
+ cursor:pointer;
+ color:#ffffff;
+ font-family:Arial;
+ font-size:20px;
+ padding:8px 16px;
+ text-decoration:none;
+}
+.myButton:hover {
+ background:#396bd1;
+ background-color:#396bd1;
+ color: #e0e0e0;
+ text-decoration: none;
+}
+.myButton:active {
+ position:relative;
+ top:1px;
+}
+</style>
diff --git a/examples/example_readme.py b/examples/example_readme.py
index 197ada6ca723871aaa24aaa87b3aed6469311d28..6798f9dd2e6916cc8b9354f8f68898247f858f37 100644
--- a/examples/example_readme.py
+++ b/examples/example_readme.py
@@ -7,32 +7,38 @@ def create_example_file(path):
f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0)
f.attrs["experiment_setup"] = "This is a simple example with made up data."
- # Create a new robot entity. Positions and orientations are passed
- # separately in this example. Since the orientations have two columns,
- # unit vectors are assumed (orientation_x, orientation_y)
- circle_rad = np.linspace(0, 2 * np.pi, num=100)
+ # Create a new robot entity with 10 timesteps.
+ # Positions and orientations are passed separately in this example.
+ # Since the orientations have two columns, unit vectors are assumed
+ # (orientation_x, orientation_y)
f.create_entity(
category="robot",
name="robot",
- positions=np.stack((np.cos(circle_rad), np.sin(circle_rad))).T * 40,
- orientations=np.stack((-np.sin(circle_rad), np.cos(circle_rad))).T,
+ positions=np.zeros((10, 2)),
+ orientations=np.ones((10, 2)) * [0, 1],
)
- # Create a new fish entity.
+ # Create a new fish entity with 10 timesteps.
# In this case, we pass positions and orientations together (x, y, rad).
# Since it is a 3 column array, orientations in radiants are assumed.
- poses = np.zeros((100, 3))
- poses[:, 0] = np.arange(-50, 50)
- poses[:, 1] = np.arange(-50, 50)
- poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100)
+ poses = np.zeros((10, 3))
+ poses[:, 0] = np.arange(-5, 5)
+ poses[:, 1] = np.arange(-5, 5)
+ poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10)
fish = f.create_entity("fish", poses=poses)
fish.attrs["species"] = "My rotating spaghetti fish"
fish.attrs["fish_standard_length_cm"] = 10
- # Show and save the file
- print(f)
- print("Poses Shape: ", f.entity_poses.shape)
+ # Some possibilities to access the data
+ print(f"The file:\n{f}")
+ print(
+ f"Poses Shape:\t{f.entity_poses_rad.shape}.\t"
+ + "Representing(entities, timesteps, pose dimensions (x, y, ori)"
+ )
+ print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}")
+ print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}")
+ # Save the file
f.save_as(path)
diff --git a/setup.py b/setup.py
index 552f54e96c461e588672a3238c82b98cee1b8030..727eb550d2f3c26bc919e1b9379a75a45f98ec1d 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ setup(
version=source_version(),
author="",
author_email="",
- install_requires=["h5py>=2.10.0", "numpy", "seaborn", "pandas", "deprecation"],
+ install_requires=["h5py>=3.1.0", "numpy", "seaborn", "pandas", "deprecation"],
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Science/Research",
diff --git a/src/robofish/evaluate/app.py b/src/robofish/evaluate/app.py
index d7cd096449edac41d97e3bcbd0519cfc281b547d..e1bf0b1a8c46c57c274084d6927dab6965f5e432 100644
--- a/src/robofish/evaluate/app.py
+++ b/src/robofish/evaluate/app.py
@@ -43,24 +43,26 @@ def evaluate(args=None):
fdict = function_dict()
+ longest_name = max([len(k) for k in fdict.keys()])
+
parser = argparse.ArgumentParser(
- description="This function can be called from the commandline to evaluate files.\
- Different evaluation methods can be called, which generate graphs from the given files. \
- \
- With the first argument 'analysis_type', the type of analysis is chosen."
+ description="This function can be called from the commandline to evaluate files.\n"
+ + "Different evaluation methods can be called, which generate graphs from the given files.\n"
+ + "With the first argument 'analysis_type', the type of analysis is chosen.",
+ formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"analysis_type",
type=str,
choices=fdict.keys(),
- help="The type of analysis.\
- speed - A histogram of speeds\
- turn - A histogram of angular velocities\
- tank_positions - A heatmap of the positions in the tank\
- trajectories - A plot of all the trajectories\
- follow_iid - A plot of the follow metric in relation to iid (inter individual distance)\
- ",
+ help="The type of analysis.\n"
+ + "\n".join(
+ [
+ f"{key}{' ' * (longest_name - len(key))} - {func.__doc__.splitlines()[0]}"
+ for key, func in fdict.items()
+ ]
+ ),
)
parser.add_argument(
"paths",
diff --git a/src/robofish/evaluate/evaluate.py b/src/robofish/evaluate/evaluate.py
index 136fafaba653ad06162195e005a5a03251bafa80..1486f06b51b808d6e21d25af1220e51076e5a02b 100644
--- a/src/robofish/evaluate/evaluate.py
+++ b/src/robofish/evaluate/evaluate.py
@@ -124,6 +124,7 @@ def evaluate_turn(
for k, files in enumerate(files_per_path):
path_turns = []
for p, file in files.items():
+ # TODO: Use new io functions (entity_poses_calc_ori_rad)
poses = file.select_entity_poses(
None if predicate is None else predicate[k]
)
diff --git a/src/robofish/io/app.py b/src/robofish/io/app.py
index fc6fee0f27766b179fa494547b67cee736cd7083..e31e9ca38a8584312ec8250c6bb733a55b03c479 100644
--- a/src/robofish/io/app.py
+++ b/src/robofish/io/app.py
@@ -1,10 +1,10 @@
# -*- coding: utf-8 -*-
+"""
+.. include:: ../../../docs/app.md
+"""
+
# -----------------------------------------------------------
-# Functions available to be used in the commandline to show and validate hdf5
-# according to the Robofish track format (1.0 Draft 7). The standard is
-# available at https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format
-#
# Dec 2020 Andreas Gerken, Berlin, Germany
# Released under GNU 3.0 License
# email andi.gerken@gmail.com
diff --git a/src/robofish/io/entity.py b/src/robofish/io/entity.py
index 5e8898abb893fd1f799bf1049920442fa14b663e..b3b7b219e52379b0d2f0355828538bb3c2d51202 100644
--- a/src/robofish/io/entity.py
+++ b/src/robofish/io/entity.py
@@ -1,3 +1,7 @@
+"""
+.. include:: ../../../docs/entity.md
+"""
+
import robofish.io
import robofish.io.utils as utils
@@ -134,24 +138,45 @@ class Entity(h5py.Group):
def orientations(self):
if not "orientations" in self:
# If no orientation is given, the default direction is to the right
- return np.tile([1, 0], (self.positions.shape[0], 1))
+ return np.tile([0, 1], (self.positions.shape[0], 1))
return self["orientations"]
+ @property
+ def orientations_rad(self):
+ ori_rad = utils.limit_angle_range(
+ np.arctan2(self.orientations[:, 1], self.orientations[:, 0]),
+ _range=(0, 2 * np.pi),
+ )
+ return ori_rad[:, np.newaxis]
+
+ @property
+ def poses_calc_ori_rad(self):
+ # Diff between positions [t - 1, 2]
+ diff = np.diff(self.positions, axis=0)
+
+ # angles [t - 1]
+ angles = utils.limit_angle_range(
+ np.arctan2(diff[:, 1], diff[:, 0]), _range=(0, 2 * np.pi)
+ )
+
+ # Positions with angles. The first position is cut of, as it does not have an orientation.
+ poses_with_calculated_orientation = np.concatenate(
+ [self.positions[1:], angles[:, np.newaxis]], axis=1
+ )
+
+ return poses_with_calculated_orientation
+
@property
def poses(self):
return np.concatenate([self.positions, self.orientations], axis=1)
@property
def poses_rad(self):
- poses = self.poses
- # calculate the angles from the orientation vectors, write them to the third row and delete the fourth row
- poses[:, 2] = np.arctan2(poses[:, 3], poses[:, 2])
- poses = poses[:, :3]
- return poses
+ return np.concatenate([self.positions, self.orientations_rad], axis=1)
@property
- def speed_turn_angle(self):
- """Get the speed, turn and angles from the positions.
+ def speed_turn(self):
+ """Get the speed, turn and from the positions.
The vectors pointing from each position to the next are computed.
The output of the function describe these vectors.
@@ -160,13 +185,14 @@ class Entity(h5py.Group):
The first column is the length of the vectors.
The second column is the turning angle, required to get from one vector to the next.
We assume, that the entity is oriented "correctly" in the first pose. So the first turn angle is 0.
- The third column is the orientation of each vector.
"""
- diff = np.diff(self.positions, axis=0)
- speed = np.linalg.norm(diff, axis=1)
- angles = np.arctan2(diff[:, 1], diff[:, 0])
- turn = np.zeros_like(angles)
- turn[0] = 0
- turn[1:] = utils.limit_angle_range(np.diff(angles))
- return np.stack([speed, turn, angles], axis=-1)
\ No newline at end of file
+ # poses with calulated orientation have first position cut of as it does not have an orientation
+ # (t - 1, (x ,y, ori))
+ poses_calc_ori = self.poses_calc_ori_rad
+
+ # Differences cuts of last item (t - 2, (dx, dy, d ori))
+ diff = np.diff(poses_calc_ori, axis=0)
+ speed = np.linalg.norm(diff[:, :2], axis=1)
+ turn = utils.limit_angle_range(diff[:, 2], _range=(-np.pi, np.pi))
+ return np.stack([speed, turn], axis=-1)
diff --git a/src/robofish/io/file.py b/src/robofish/io/file.py
index e8fcfc2256507b53e1e4d178b524b780f8e5428f..5d5742ffd3f28d63e038178db0ab296927e6249e 100644
--- a/src/robofish/io/file.py
+++ b/src/robofish/io/file.py
@@ -1,13 +1,14 @@
# -*- coding: utf-8 -*-
+"""
+.. include:: ../../../docs/file.md
+"""
+
# -----------------------------------------------------------
# Utils functions for reading, validating and writing hdf5 files according to
# Robofish track format (1.0 Draft 7). The standard is available at
# https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format
-#
-# The term track is used to describe a dictionary, describing the track in a dict.
-# To distinguish between attributes, dictionaries and groups, a prefix is used
-# (a_ for attribute, d_ for dictionary, and g_ for groups).
+
#
# Dec 2020 Andreas Gerken, Berlin, Germany
# Released under GNU 3.0 License
@@ -30,6 +31,7 @@ import uuid
import deprecation
import types
+# Remember: Update docstring when updating these two global variables
default_format_version = np.array([1, 0], dtype=np.int32)
default_format_url = (
@@ -61,26 +63,64 @@ class File(h5py.File):
):
"""Create a new RoboFish Track Format object.
- When called with a path, it is loaded, otherwise a new temporary file is created.
+ When called with a path, it is loaded, otherwise a new temporary
+ file is created. File contents can be validated against the
+ track format specification.
Parameters
----------
path : str or Path, optional
Location of file to be opened. If not provided, mode is ignored.
-
- mode : str
- r Readonly, file must exist (default)
- r+ Read/write, file must exist
- w Create file, truncate if exists
- x Create file, fail if exists
- a Read/write if exists, create otherwise
-
- world_size_cm
- optional integer array of the world size in cm
- strict_validate
- optional boolean, if the file should be strictly validated, when loaded from a path. The default is False.
- format_version
- optional version [major, minor] of the trackformat specification
+ mode : str, default='r'
+ 'r' Readonly, file must exist
+ 'r+' Read/write, file must exist
+ 'w' Create file, truncate if exists
+ 'x' Create file, fail if exists
+ 'a' Read/write if exists, create otherwise
+ world_size_cm : [int, int] , optional
+ side lengths [x, y] of the world in cm.
+ rectangular world shape is assumed.
+ TODO: Cuboid world is also possible in track format
+ strict_validate : bool, default=False
+ if the file should be strictly validated against the track
+ format specification, when loaded from a path.
+ TODO: Should this validate against the version sepcified in
+ format_version or just against the most recent version?
+ format_version : [int, int], default=[1,0]
+ version [major, minor] of track format specification
+ format_url : str, default="https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/-/releases/1.0"
+ location of track format specification.
+ should fit `format_version`.
+ sampling_name : str, optional
+ How to specify your sampling:
+
+ 1. (optional)
+ provide text description of your sampling in `sampling_name`
+
+ 2.a (mandatory, if you have a constant sampling frequency)
+ specify `frequency_hz` with your sampling frequency in Hz
+
+ 2.b (mandatory, if you do NOT have a constant sampling frequency)
+ specify `monotonic_time_points_us` with a list[1] of time
+ points in microseconds on a montonic clock, one for each
+ sample in your dataset.
+
+ 3. (optional)
+ specify `calendar_time_points` with a list[2] of time points
+ in the ISO 8601 extended format with microsecond precision
+ and time zone designator[3], one for each sample in your
+ dataset.
+
+ [1] any Iterable of int
+ [2] any Iterable of str
+ [3] example: "2020-11-18T13:21:34.117015+01:00"
+
+ frequency_hz: int, optional
+ refer to explanation of `sampling_name`
+ monotonic_time_points_us: Iterable of int, optional
+ refer to explanation of `sampling_name`
+ calendar_time_points: Iterable of str, optional
+ refer to explanation of `sampling_name`
"""
if path is None:
@@ -143,6 +183,8 @@ class File(h5py.File):
Args:
path: path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used.
strict_validate: optional boolean, if the file should be strictly validated, before saving. The default is True.
+ Returns:
+ The file itself, so something like f = robofish.io.File().save_as("file.hdf5") works
"""
self.validate(strict_validate=strict_validate)
@@ -155,6 +197,8 @@ class File(h5py.File):
shutil.copyfile(Path(self.filename).resolve(), path)
+ return self
+
def create_sampling(
self,
name: str = None,
@@ -319,7 +363,6 @@ class File(h5py.File):
assert poses.ndim == 3
assert poses.shape[2] in [3, 4]
agents = poses.shape[0]
- timesteps = poses.shape[1]
entity_names = []
for i in range(agents):
@@ -355,20 +398,38 @@ class File(h5py.File):
for name in self.entity_names
]
+ @property
+ def entity_positions(self):
+ return self.select_entity_property(None, entity_property=Entity.positions)
+
+ @property
+ def entity_orientations(self):
+ return self.select_entity_property(None, entity_property=Entity.orientations)
+
+ @property
+ def entity_orientations_rad(self):
+ return self.select_entity_property(
+ None, entity_property=Entity.orientations_rad
+ )
+
@property
def entity_poses(self):
- return self.select_entity_property(None)
+ return self.select_entity_property(None, entity_property=Entity.poses)
@property
def entity_poses_rad(self):
return self.select_entity_property(None, entity_property=Entity.poses_rad)
@property
- def speeds_turns_angles(self):
+ def entity_poses_calc_ori_rad(self):
return self.select_entity_property(
- None, entity_property=Entity.speed_turn_angle
+ None, entity_property=Entity.poses_calc_ori_rad
)
+ @property
+ def entity_speeds_turns(self):
+ return self.select_entity_property(None, entity_property=Entity.speed_turn)
+
def select_entity_poses(self, *args, ori_rad=False, **kwargs):
entity_property = Entity.poses_rad if ori_rad else Entity.poses
return self.select_entity_property(
diff --git a/src/robofish/io/io.py b/src/robofish/io/io.py
index ffe559c843b370027e9e145317b2fe478171e742..4c12aded73ffab9675370b160dac54b832164331 100644
--- a/src/robofish/io/io.py
+++ b/src/robofish/io/io.py
@@ -71,28 +71,34 @@ def read_multiple_files(
return sf_dict
-def read_poses_from_multiple_files(
+def read_property_from_multiple_files(
paths: Union[Path, str, Iterable[Path], Iterable[str]],
+ entity_property: property = None,
+ *,
strict_validate: bool = False,
max_files: int = None,
shuffle: bool = False,
- ori_rad: bool = False,
+ predicate: callable = None,
):
- """Load hdf5 files from a given path and return the entity poses.
+ """Load hdf5 files from a given path and return the property of the entities.
The function can be given the path to a single single hdf5 file, to a folder,
containing hdf5 files, or an array of multiple files or folders.
Args:
path: The path to a hdf5 file or folder.
+ entity_property: A property of robofish.io.Entity default is Entity.poses_rad
strict_validate: Choice between error and warning in case of invalidity
max_files: Maximum number of files to be read
shuffle: Shuffle the order of files
- ori_rad: Return the orientations as radiants instead of unit vectors
+ predicate:
Returns:
- An array of all entity poses arrays
+ An array of all entity properties arrays
"""
+ assert (
+ entity_property is not None
+ ), "Please select an entity property e.g. 'Entity.poses_rad'"
logging.info(f"Reading files from path {paths}")
list_types = (list, np.ndarray, pandas.core.series.Series)
@@ -122,12 +128,12 @@ def read_poses_from_multiple_files(
with robofish.io.File(
path=file, strict_validate=strict_validate
) as f:
- p = f.entity_poses_rad if ori_rad else f.entity_poses
+ p = f.select_entity_property(predicate, entity_property)
poses_array.append(p)
elif path is not None and path.exists():
logging.info("found file %s" % path)
with robofish.io.File(path=path, strict_validate=strict_validate) as f:
- p = f.entity_poses_rad if ori_rad else f.entity_poses
+ p = f.select_entity_property(predicate, entity_property)
poses_array.append(p)
return poses_array
\ No newline at end of file
diff --git a/tests/robofish/io/test_entity.py b/tests/robofish/io/test_entity.py
index 3f3c4203ae08fd98ece2bff1b1a55f4f268d6c39..72dc5f7efe8a0ed1296a66f39473e575230dd123 100644
--- a/tests/robofish/io/test_entity.py
+++ b/tests/robofish/io/test_entity.py
@@ -39,21 +39,20 @@ def test_entity_turn_speed():
[np.cos(circle_rad) * circle_size, np.sin(circle_rad) * circle_size], axis=-1
)
e = f.create_entity("fish", positions=positions)
- speed_turn_angle = e.speed_turn_angle
- assert speed_turn_angle.shape == (99, 3)
+ speed_turn = e.speed_turn
+ assert speed_turn.shape == (98, 2)
- # No turn in the first timestep, since initialization turns it the right way
- assert speed_turn_angle[0, 1] == 0
+ # Turns and speeds shoud be all the same afterwards, since the fish swims with constant velocity and angular velocity.
+ assert (np.std(speed_turn, axis=0) < 0.0001).all()
- # Turns and speeds shoud afterwards be all the same afterwards, since the fish swims with constant velocity and angular velocity.
- assert (np.std(speed_turn_angle[1:, :2], axis=0) < 0.0001).all()
+ # Cut off the first position as it cannot be simulated
+ positions = positions[1:]
# Use turn_speed to generate positions
gen_positions = np.zeros((positions.shape[0], 3))
- gen_positions[0, :2] = positions[0]
- gen_positions[0, 2] = speed_turn_angle[0, 2]
+ gen_positions[0] = e.poses_calc_ori_rad[0]
- for i, (speed, turn, angle) in enumerate(speed_turn_angle):
+ for i, (speed, turn) in enumerate(speed_turn):
new_angle = gen_positions[i, 2] + turn
gen_positions[i + 1] = [
gen_positions[i, 0] + np.cos(new_angle) * speed,
diff --git a/tests/robofish/io/test_file.py b/tests/robofish/io/test_file.py
index 8e4c4a6b625611f1cea33f21c1810e512cea3419..31b3bda15fd51f3ba87b988fa488ff8c2f218f53 100644
--- a/tests/robofish/io/test_file.py
+++ b/tests/robofish/io/test_file.py
@@ -143,7 +143,9 @@ def test_speeds_turns_angles():
with robofish.io.File(world_size_cm=[100, 100], frequency_hz=25) as f:
poses = np.zeros((10, 100, 3))
f.create_multiple_entities("fish", poses=poses)
- assert (f.speeds_turns_angles == 0).all()
+
+ # Stationary fish has no speed or turn
+ assert (f.entity_speeds_turns == 0).all()
def test_broken_sampling(caplog):
@@ -182,8 +184,13 @@ def test_entity_positions_no_orientation():
# Create an entity, using radiants
f.create_entity("fish", positions=np.ones((100, 2)))
+ # In poses, the default orientation pointing up should be added.
assert f.entity_poses.shape == (1, 100, 4)
- assert (f.entity_poses[:, :] == np.array([1, 1, 1, 0])).all()
+ assert (f.entity_poses[:, :] == np.array([1, 1, 0, 1])).all()
+ assert np.isclose(f.entity_orientations_rad, np.pi / 2).all()
+
+ # Calculate the orientation
+ assert f.entity_poses_calc_ori_rad.shape == (1, 99, 3)
def test_load_validate():
diff --git a/tests/robofish/io/test_io.py b/tests/robofish/io/test_io.py
index f0449b55128a47b71b28346edf2427486a34a310..3a7bb32390223adfbd4e7f2f1f9a3454c93bdfa4 100644
--- a/tests/robofish/io/test_io.py
+++ b/tests/robofish/io/test_io.py
@@ -45,8 +45,10 @@ path = utils.full_path(__file__, "../../resources/valid.hdf5")
# TODO read from folder of valid files
@pytest.mark.parametrize("_path", [path, str(path)])
-def test_read_poses_from_multiple_folder(_path):
- poses = robofish.io.read_poses_from_multiple_files([_path, _path])
+def test_read_poses_rad_from_multiple_folder(_path):
+ poses = robofish.io.read_property_from_multiple_files(
+ [_path, _path], robofish.io.entity.Entity.poses_rad
+ )
# Should find the 3 presaved hdf5 files
assert len(poses) == 2
for p in poses: