diff --git a/.gitignore b/.gitignore index 5f762743f6afcdf7443b26c2a4ee3288102fa82b..3825a6ab5a89436ea58a3513859272e88abee742 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ dist .coverage report.xml htmlcov -docs +html env !tests/resources/*.hdf5 diff --git a/README.md b/README.md index 5da734e05530add0c7ef723c9990f3634dbcd57f..8e6eafc69af1da3141a0e49008a81771e199e4ca 100644 --- a/README.md +++ b/README.md @@ -9,59 +9,60 @@ [](https://git.imp.fu-berlin.de/bioroboticslab/robofish/io/commits/master) # Robofish IO -This repository implements an easy to use interface, to create, save, load, and work [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files. +This repository implements an easy to use interface, to create, save, load, and work with [specification-compliant](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format) hdf5 files, containing 2D swarm data. This repository should be used by the different swarm projects to generate comparable standardized files. +## <a href="http://agerken.de/io/index.html">Documentation</a> ## Installation -Quick variant: -``` -pip3 install robofish-trackviewer robofish-io --extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple -``` - -Better variant: -- Follow instructions at [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) -- ```pip3 install robofish-io``` +Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) to your pip config and install the packagage. +```bash +python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple +python3 -m pip install robofish-io +``` ## Usage We show a simple example below. More examples can be found in ```examples/``` ```python -import robofish.io -import numpy as np - - # Create a new robofish io file f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) f.attrs["experiment_setup"] = "This is a simple example with made up data." -# Create a new robot entity. Positions and orientations are passed -# separately in this example. Since the orientations have two columns, -# unit vectors are assumed (orientation_x, orientation_y) +# Create a new robot entity with 10 timesteps. +# Positions and orientations are passed separately in this example. +# Since the orientations have two columns, unit vectors are assumed +# (orientation_x, orientation_y) f.create_entity( category="robot", name="robot", - positions=np.zeros((100, 2)), - orientations=np.ones((100, 2)) * [0, 1], + positions=np.zeros((10, 2)), + orientations=np.ones((10, 2)) * [0, 1], ) -# Create a new fish entity. +# Create a new fish entity with 10 timesteps. # In this case, we pass positions and orientations together (x, y, rad). # Since it is a 3 column array, orientations in radiants are assumed. -poses = np.zeros((100, 3)) -poses[:, 0] = np.arange(-50, 50) -poses[:, 1] = np.arange(-50, 50) -poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100) +poses = np.zeros((10, 3)) +poses[:, 0] = np.arange(-5, 5) +poses[:, 1] = np.arange(-5, 5) +poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10) fish = f.create_entity("fish", poses=poses) fish.attrs["species"] = "My rotating spaghetti fish" fish.attrs["fish_standard_length_cm"] = 10 -# Show and save the file -print(f) -print("Poses Shape: ", f.entity_poses.shape) +# Some possibilities to access the data +print(f"The file:\n{f}") +print( + f"Poses Shape:\t{f.entity_poses_rad.shape}.\t" + + "Representing(entities, timesteps, pose dimensions (x, y, ori)" +) +print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}") +print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}") -f.save_as(path) +# Save the file +f.save_as("example.hdf5") ``` ### Evaluation @@ -71,14 +72,4 @@ Current modes are: - turn - tank_positions - trajectories -- follow_iid - - -## LICENSE - -This work is licensed under LGPL 3.0 (or any later version). -Individual files contain the following tag instead of the full license text: - -`SPDX-License-Identifier: LGPL-3.0-or-later` - -This enables machine processing of license information based on the SPDX License Identifiers available here: https://spdx.org/licenses/ +- follow_iid \ No newline at end of file diff --git a/docs/app.md b/docs/app.md new file mode 100644 index 0000000000000000000000000000000000000000..54bb225a100dd4f7311d1e7fa4abb5b7dddf6248 --- /dev/null +++ b/docs/app.md @@ -0,0 +1,99 @@ +This module defines the command line interface. +All commands have a ``--help`` option to get more info about them in the command line. + + +## Print +```bash +robofish-io-print example.hdf5 +``` +Checking out the content of files. + +<small> + +``` +usage: robofish-io-print [-h] [--output_format {shape,full}] path + +This function can be used to print hdf5 files from the command line +positional arguments: + path The path to a hdf5 file +optional arguments: + -h, --help show this help message and exit + --output_format {shape,full} + Choose how datasets are printed, either the shapes or the full content is printed +``` + +</small> + +## Evaluate +```bash +robofish-io-evaluate *analysis_type* example.hdf5 +``` +Show some property of a file. + + +<small> + +``` +usage: robofish-io-evaluate [-h] [--names NAMES [NAMES ...]] [--save_path SAVE_PATH] + {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid} paths [paths ...] + +This function can be called from the commandline to evaluate files. +Different evaluation methods can be called, which generate graphs from the given files. +With the first argument 'analysis_type', the type of analysis is chosen. + +positional arguments: + {speed,turn,orientation,relative_orientation,distance_to_wall,tank_positions,trajectories,evaluate_positionVec,follow_iid} + The type of analysis. + speed - Evaluate the speed of the entities as histogram. + turn - Evaluate the turn angles of the entities as histogram. + orientation - Evaluate the orientations of the entities on a 2d grid. + relative_orientation - Evaluate the relative orientations of the entities as a histogram. + distance_to_wall - Evaluate the distances of the entities to the walls as a histogram. + tank_positions - Evaluate the positions of the entities as a heatmap. + trajectories - Evaluate the trajectories of the entities. + evaluate_positionVec - Evaluate the vectors pointing from the focal fish to the conspecifics as heatmap. + follow_iid - Evaluate the follow metric in respect to the inter individual distance (iid). + paths The paths to files or folders. Multiple paths can be given to compare experiments. + +optional arguments: + -h, --help show this help message and exit + --names NAMES [NAMES ...] + Names, that should be used in the graphs instead of the pahts. + --save_path SAVE_PATH + Filename for saving resulting graphics. +``` + +</small> + +## Trackviewer + +```bash +robofish-trackviewer example.hdf5 +``` + +The trackviewer is from a different repository. It was included in the install instructions. + +<small> + +``` +usage: robofish-trackviewer [-h] [--draw-labels] [--draw-view-vectors] [--far-plane FAR_PLANE] [--view-of-agents field of perception number of bins] [--view-of-walls field of perception number of bins] + [--view-of-walls-matches] + [trackset_file] + +View RoboFish tracks in a GUI. + +positional arguments: + trackset_file Path to HDF5 file containing the tracks to view + +optional arguments: + -h, --help show this help message and exit + --draw-labels Whether to draw labels inside the agents' outlines + --draw-view-vectors Whether to draw view vectors to the right of / below the trackfile + --far-plane FAR_PLANE + Maximum distance an agent can see + --view-of-agents field of perception number of bins + --view-of-walls field of perception number of bins + --view-of-walls-matches +``` + +</small> \ No newline at end of file diff --git a/docs/entity.md b/docs/entity.md new file mode 100644 index 0000000000000000000000000000000000000000..58b5f8606ae9992adf49efb74b8f3363ea5f9c1b --- /dev/null +++ b/docs/entity.md @@ -0,0 +1,88 @@ +Tracks of entities are stored in `robofish.io.entity.Entity` objects. They are created by the `robofish.io.file.File` object. They require a category and can have a name. If no name is given, it will be created, using the category and an id. + +```python +f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) +nemo = f.create_entity(category="fish", name="nemo") +``` + +The function `create_entity()` returns an entity object. It can be stored but it's not neccessary. The entity is automatically stored in the file. + +## Pose options + +The poses of an entity can be passed in multiple ways. Poses are divided into `positions` (x, y) and `orientations`. +Orientations are internally represented in unit vectors but can also be passed in rads. +The shape of the passed orientation array defines the meaning. + +```python +# Create dummy poses for 100 timesteps +pos = np.zeros((100, 2)) +ori_vec = np.zeros((100,2)) * [0, 1] +ori_rad = np.zeros((100,1)) + +# Creating an entity without orientations. Here we keep the entity object, to use it later. +f.create_entity(category="fish", positions=pos) +# Creating an entity using orientation vectors. Keeping the entity object is not neccessary, it is saved in the file. +f.create_entity(category="fish", positions=pos, orientations=ori_vec) +# Creating an entity using radiant orientations. +f.create_entity(category="fish", positions=pos, orientations=ori_rad) +``` + +The poses can be also passed in an combined array. +```python +# Create an entity using orientation vectors. +f.create_entity(category="fish", poses=np.ones((100,4)) * np.sqrt(2)) +# Create an entity using radiant orientations +f.create_entity(category="fish", poses=np.zeros((100,3))) +``` + +## Creating multiple entities at once + +Multiple entities can be created at once. +```python +# Here we create 4 fishes from poses with radiant orientation +f.create_multiple_entities(category="fish", poses=np.zeros((4,100,3))) +``` + +## Attributes + +Entities can have attributes to describe them. + +The attributes can be set like this: +```python +nemo.attrs["species"] = "Clownfish" +nemo.attrs["fish_standard_length_cm"] = 10 +``` + +Any attribute is allowed, but some cannonical attributes are prepared:<br> +`species`:`str`, `sex`:`str`, `fish_standard_length_cm`:`float` + + +## Properties + +As described in `robofish.io`, Files and Entities have useful properties. + +<small> + +| Entity function | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------- | +| `robofish.io.entity.Entity.positions` | Positions as a `(timesteps, 2 (x, y))` arary. | +| `robofish.io.entity.Entity.orientations` | Orientations as a `(timesteps, 2 (ori_x, ori_y))` arary. | +| `robofish.io.entity.Entity.orientations_rad` | Orientations as a `(timesteps, 1 (ori_rad))` arary. | +| `robofish.io.entity.Entity.poses` | Poses as a `(timesteps, 4 (x, y, x_ori, y_ori))` array. | +| `robofish.io.entity.Entity.poses_rad` | Poses as a `(timesteps, 3(x, y, ori_rad))` array. | +| `robofish.io.entity.Entity.poses_calc_ori_rad` | Poses with calculated orientations as a<br>`(timesteps - 1, 3 (x, y, calc_ori_rad))` array. | +| `robofish.io.entity.Entity.speed_turn` | Speed and turn as a `(timesteps - 2, 2 (speed_cm/s, turn_rad/s))` array. | + +</small> + +### Calculated orientations and speeds. + + + +The image shows, how the orientations are calculated (`robofish.io.entity.Entity.poses_calc_ori_rad`), the orientations always show away from the last position. In this way, the first position does not have an orientation and the shape of the resulting array is `(timesteps - 1, 3 (x, y, calc_ori_rad))`. + +The meaning of the `robofish.io.entity.Entity.speed_turn` can also be explained with the image. The turn is the angle, the agent has to turn, to orientate towards the next position. The speed is the calculated, using the distance between two positions. The resulting turn and speed is converted to `[rad/s]` and `[cm/s]`. The first and last position don't have any action which results in an array shape of `(timesteps - 2, 2 (speed, turn))`. + +--- + +⚠️ Try this out by extending the example of the main doc, so that a new teleporting fish with random positions [-50, 50] is generated. How does that change the output and speed histogram `robofish-io-evaluate speed example.hdf5`? Try writing some attributes. diff --git a/docs/file.md b/docs/file.md new file mode 100644 index 0000000000000000000000000000000000000000..bdfbfba3cca6d8201b8b8e0ec5983f5b8670dfab --- /dev/null +++ b/docs/file.md @@ -0,0 +1,45 @@ +`robofish.io.file.File` objects are the root of the project. The object contains all information about the environment, entities, and time. + +In the simplest form we define a new File with a world size in cm and a frequency in hz. Afterwards, we can save it with a path. + +```python +import robofish.io + +# Create a new robofish io file +f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) +f.save_as("test.hdf5") +``` + +The File object can also be generated, with a given path. In this case, we work on the file directly. The `with` block ensures, that the file is validated after the block. + +```python +with robofish.io.File( + "test.hdf5", mode="x", world_size_cm=[100, 100], frequency_hz=25.0 +) as f: + # Use file f here +``` + +When opening a file with a path, a mode should be specified to describe how the file should be opened. + +| Mode | Description | +|------ |---------------------------------------- | +| r | Readonly, file must exist (default) | +| r+ | Read/write, file must exist | +| w | Create file, truncate if exists | +| x | Create file, fail if exists | +| a | Read/write if exists, create otherwise | + +## Attributes + +Attributes of the file can be added, to describe the contents. +The attributes can be set like this: +```python +f.attrs["experiment_setup"] = "This file comes from the tutorial." +f.attrs["experiment_issues"] = "All data in this file is made up." +``` + +Any attribute is allowed, but some cannonical attributes are prepared:<br> +`publication_url, video_url, tracking_software_name, tracking_software_version, tracking_software_url, experiment_setup, experiment_issues` + +## Properties +As described in `robofish.io`, all properties of `robofish.io.entity`s can be accessed by adding the prefix `entity_` to the function. \ No newline at end of file diff --git a/docs/img/calc_ori_speed_turn.png b/docs/img/calc_ori_speed_turn.png new file mode 100644 index 0000000000000000000000000000000000000000..439d4c95455a2c2ee8f30ee616ccf2420277a5e3 Binary files /dev/null and b/docs/img/calc_ori_speed_turn.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000000000000000000000000000000000..df674a31b092d893994a20961bf90ebcebbaf5ea --- /dev/null +++ b/docs/index.md @@ -0,0 +1,191 @@ +This package provides you: + +- Creation, storing, loading, modifying, inspecting of io-files. +- Preprocessing (orientation calculation, action calculation, raycasting, ...) +- Quick and easy evaluation of behavior +- Data, which is interchangable between labs and tools. No conversions required, since units and coordinate systems are standardized. +- No custom data import, just `include robofish.io` +- This package is tested extensively + +Features coming up: + +- Interface for unified behavior models +- Pytorch Datasets directly from `robofish.io` files. + + +<a href="https://git.imp.fu-berlin.de/bioroboticslab/robofish/io" class="myButton">Code</a> + +## Installation +Add our [Artifacts repository](https://git.imp.fu-berlin.de/bioroboticslab/robofish/artifacts) to your pip config and install the packagage. + +```bash +python3 -m pip config set global.extra-index-url https://git.imp.fu-berlin.de/api/v4/projects/6392/packages/pypi/simple +python3 -m pip install robofish-io robofish-trackviewer +``` + +## Usage + +This documentation is structured with increasing complexity. +First we'll execute the example from the readme. More examples can be found in ```examples/```. + +```python +# Create a new robofish io file +f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) +f.attrs["experiment_setup"] = "This is a simple example with made up data." + +# Create a new robot entity with 10 timesteps. +# Positions and orientations are passed separately in this example. +# Since the orientations have two columns, unit vectors are assumed +# (orientation_x, orientation_y) +f.create_entity( + category="robot", + name="robot", + positions=np.zeros((10, 2)), + orientations=np.ones((10, 2)) * [0, 1], +) + +# Create a new fish entity with 10 timesteps. +# In this case, we pass positions and orientations together (x, y, rad). +# Since it is a 3 column array, orientations in radiants are assumed. +poses = np.zeros((10, 3)) +poses[:, 0] = np.arange(-5, 5) +poses[:, 1] = np.arange(-5, 5) +poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10) +fish = f.create_entity("fish", poses=poses) +fish.attrs["species"] = "My rotating spaghetti fish" +fish.attrs["fish_standard_length_cm"] = 10 + +# Some possibilities to access the data +print(f"The file:\n{f}") +print( + f"Poses Shape:\t{f.entity_poses_rad.shape}.\t" + + "Representing(entities, timesteps, pose dimensions (x, y, ori)" +) +print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}") +print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}") + +# Save the file +f.save_as("example.hdf5") +``` + +⚠️ Please try out the example on your computer and read the output. + +We created an `robofish.io.file.File` object, then we added two `robofish.io.entity.Entity` objects. +Afterwards we read some properties of the file and printed them *(more info in [Reading Properties](#reading-properties))*. +Lastly, we saved the file to `example.hdf5`. +Congrats, you created your first io file. We'll continue working with it. + +--- + +We can examine the file now by using commandline tools. These are some examples, more details in [Commandline Tools](## Commandline Tools) + +```bash +robofish-io-print example.hdf5 +``` +Checking out the file content. + +```bash +robofish-io-evaluate speed example.hdf5 +``` +Show a histogram of speeds in the file. For more evaluation options check `robofish-io-evaluate --help` + +```bash +robofish-trackviewer example.hdf5 +``` +View a video of the track in an interactive window. + +Further details about the commandline tools can be found in `robofish.io.app`. + +## Accessing real data +Until now, we only worked with dummy data. Data from different sources is available in the Trackdb. It is currently stored at the FU Box. + +⚠️ If you don't have access to the Trackdb yet, please text Andi by Mail or Mattermost (andi.gerken@gmail.com) + + +## Reading properties + +Files and entities have usefull properties to access their content. In this way, positions, orientations, speeds, and turns can be accessed easily. + +All shown property functions can be called from a file or on one entity. +The function names are identical but have a `entity_` prefix. + +```python +f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) +nemo = f.create_entity(category='fish', name='nemo', poses=np.zeros((10,3))) +dori = f.create_entity(category='fish', name='dori', poses=np.zeros((10,3))) + + +# Get the poses of nemo. Resulting in a (10,3) array +print(nemo.poses_rad) + +# Get the poses of all entities. Resulting in a (2,10,3) array. +print(f.entity_poses_rad) +``` + +In the same scheme the following properties are available: + +<small> + +| File/ Entity function | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------- | +| *entity_*positions | Positions as a `(*entities*, timesteps, 2 (x, y))` arary. | +| *entity_*orientations | Orientations as a `(*entities*, timesteps, 2 (ori_x, ori_y))` arary. | +| *entity_*orientations_rad | Orientations as a `(*entities*, timesteps, 1 (ori_rad))` arary. | +| *entity_*poses | Poses as a `(*entities*, timesteps, 4 (x, y, x_ori, y_ori))` array. | +| *entity_*poses_rad | Poses as a `(*entities*, timesteps, 3(x, y, ori_rad))` array. | +| *entity_*poses_calc_ori_rad | Poses with calculated orientations as a<br>`(*entities*, timesteps - 1, 3 (x, y, calc_ori_rad))` array. | +| *entity_*speed_turn | Speed and turn as a `(*entities*, timesteps - 2, 2 (speed_cm/s, turn_rad/s))` array. | + +</small> + +All these functions are described in more detail in `robofish.io.entity`. + +## Where to continue? +We recommend continuing to read advanced options for `robofish.io.file`s and `robofish.io.entity`s. +Create some files, validate them, look at them in the trackviewer, evaluate them. + +If you find bugs or get stuck somewhere, please text `Andi` on Mattermost or by mail ([andi.gerken@gmail.com](mailto:andi.gerken@gmail.com)) + + +--- + +## Extended functions +In the [Track Format Specification](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/uploads/f76d86e7a629ca38f472b8f23234dbb4/RoboFish_Track_Format_-_1.0.pdf) and in this package there are more possibilities, we don't describe explicitly here. If you want to use any of the following features, please ask Andi: + +Built in robofish.io and not described: + +- Multiple sampling frequencies +- Timestamps per timestep +- Calendar points +- Outlines +- Obstacles + +Planned in Track Format but not implemented: + +- 3D Tracks + + +<style> +.myButton { + background:#3971cc; + background-color:#3971cc; + border-radius:9px; + display:inline-block; + cursor:pointer; + color:#ffffff; + font-family:Arial; + font-size:20px; + padding:8px 16px; + text-decoration:none; +} +.myButton:hover { + background:#396bd1; + background-color:#396bd1; + color: #e0e0e0; + text-decoration: none; +} +.myButton:active { + position:relative; + top:1px; +} +</style> diff --git a/examples/example_readme.py b/examples/example_readme.py index 197ada6ca723871aaa24aaa87b3aed6469311d28..6798f9dd2e6916cc8b9354f8f68898247f858f37 100644 --- a/examples/example_readme.py +++ b/examples/example_readme.py @@ -7,32 +7,38 @@ def create_example_file(path): f = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25.0) f.attrs["experiment_setup"] = "This is a simple example with made up data." - # Create a new robot entity. Positions and orientations are passed - # separately in this example. Since the orientations have two columns, - # unit vectors are assumed (orientation_x, orientation_y) - circle_rad = np.linspace(0, 2 * np.pi, num=100) + # Create a new robot entity with 10 timesteps. + # Positions and orientations are passed separately in this example. + # Since the orientations have two columns, unit vectors are assumed + # (orientation_x, orientation_y) f.create_entity( category="robot", name="robot", - positions=np.stack((np.cos(circle_rad), np.sin(circle_rad))).T * 40, - orientations=np.stack((-np.sin(circle_rad), np.cos(circle_rad))).T, + positions=np.zeros((10, 2)), + orientations=np.ones((10, 2)) * [0, 1], ) - # Create a new fish entity. + # Create a new fish entity with 10 timesteps. # In this case, we pass positions and orientations together (x, y, rad). # Since it is a 3 column array, orientations in radiants are assumed. - poses = np.zeros((100, 3)) - poses[:, 0] = np.arange(-50, 50) - poses[:, 1] = np.arange(-50, 50) - poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 100) + poses = np.zeros((10, 3)) + poses[:, 0] = np.arange(-5, 5) + poses[:, 1] = np.arange(-5, 5) + poses[:, 2] = np.arange(0, 2 * np.pi, step=2 * np.pi / 10) fish = f.create_entity("fish", poses=poses) fish.attrs["species"] = "My rotating spaghetti fish" fish.attrs["fish_standard_length_cm"] = 10 - # Show and save the file - print(f) - print("Poses Shape: ", f.entity_poses.shape) + # Some possibilities to access the data + print(f"The file:\n{f}") + print( + f"Poses Shape:\t{f.entity_poses_rad.shape}.\t" + + "Representing(entities, timesteps, pose dimensions (x, y, ori)" + ) + print(f"The actions of one Fish, (timesteps, (speed, turn)):\n{fish.speed_turn}") + print(f"Fish poses with calculated orientations:\n{fish.poses_calc_ori_rad}") + # Save the file f.save_as(path) diff --git a/setup.py b/setup.py index 552f54e96c461e588672a3238c82b98cee1b8030..727eb550d2f3c26bc919e1b9379a75a45f98ec1d 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ setup( version=source_version(), author="", author_email="", - install_requires=["h5py>=2.10.0", "numpy", "seaborn", "pandas", "deprecation"], + install_requires=["h5py>=3.1.0", "numpy", "seaborn", "pandas", "deprecation"], classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Science/Research", diff --git a/src/robofish/evaluate/app.py b/src/robofish/evaluate/app.py index 4d603f33ff39fd4b591ce33c8c607a55aa75c979..e1bf0b1a8c46c57c274084d6927dab6965f5e432 100644 --- a/src/robofish/evaluate/app.py +++ b/src/robofish/evaluate/app.py @@ -11,6 +11,7 @@ Functions available to be used in the commandline to evaluate robofish.io files. import robofish.evaluate import argparse +from pathlib import Path def function_dict(): @@ -25,6 +26,7 @@ def function_dict(): "trajectories": base.evaluate_trajectories, "evaluate_positionVec": base.evaluate_positionVec, "follow_iid": base.evaluate_follow_iid, + "all": base.evaluate_all, } @@ -41,24 +43,26 @@ def evaluate(args=None): fdict = function_dict() + longest_name = max([len(k) for k in fdict.keys()]) + parser = argparse.ArgumentParser( - description="This function can be called from the commandline to evaluate files.\ - Different evaluation methods can be called, which generate graphs from the given files. \ - \ - With the first argument 'analysis_type', the type of analysis is chosen." + description="This function can be called from the commandline to evaluate files.\n" + + "Different evaluation methods can be called, which generate graphs from the given files.\n" + + "With the first argument 'analysis_type', the type of analysis is chosen.", + formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument( "analysis_type", type=str, choices=fdict.keys(), - help="The type of analysis.\ - speed - A histogram of speeds\ - turn - A histogram of angular velocities\ - tank_positions - A heatmap of the positions in the tank\ - trajectories - A plot of all the trajectories\ - follow_iid - A plot of the follow metric in relation to iid (inter individual distance)\ - ", + help="The type of analysis.\n" + + "\n".join( + [ + f"{key}{' ' * (longest_name - len(key))} - {func.__doc__.splitlines()[0]}" + for key, func in fdict.items() + ] + ), ) parser.add_argument( "paths", @@ -86,6 +90,11 @@ def evaluate(args=None): args = parser.parse_args() if args.analysis_type in fdict: - fdict[args.analysis_type](args.paths, args.names, args.save_path) + params = (args.paths, args.names, Path(args.save_path)) + if args.analysis_type == "all": + normal_functions = function_dict() + normal_functions.pop("all") + params += (normal_functions,) + fdict[args.analysis_type](*params) else: print(f"Evaluation function not found {args.analysis_type}") diff --git a/src/robofish/evaluate/evaluate.py b/src/robofish/evaluate/evaluate.py index a5719a5037c76b7ce76256d9481ea5ad1e342824..68fd70a97744c0953d0621183085efde2ba36ed8 100644 --- a/src/robofish/evaluate/evaluate.py +++ b/src/robofish/evaluate/evaluate.py @@ -17,6 +17,7 @@ import numpy as np import pandas as pd from typing import Iterable from scipy import stats +from tqdm import tqdm def get_all_poses_from_paths(paths: Iterable[str]): @@ -123,6 +124,7 @@ def evaluate_turn( for k, files in enumerate(files_per_path): path_turns = [] for p, file in files.items(): + # TODO: Use new io functions (entity_poses_calc_ori_rad) poses = file.select_entity_poses( None if predicate is None else predicate[k] ) @@ -676,7 +678,8 @@ def evaluate_follow_iid( def evaluate_all( paths: Iterable[str], labels: Iterable[str] = None, - save_folder: str = None, + save_folder: Path = None, + fdict: dict = None, predicate=None, ): """Generate all evaluation graphs and save them to a folder. @@ -686,23 +689,15 @@ def evaluate_all( labels: Labels for the paths. If no labels are given, the paths will be used save_path: A path to a save location. + fdict: A dictionary of strings and evaluation functions predicate: a lambda function, selecting entities (example: lambda e: e.category == "fish") """ - # save_folder = Path(save_folder) - evaluate_speed(paths, labels, save_folder + "speed.png", predicate) - evaluate_turn(paths, labels, save_folder + "turn.png", predicate) - evaluate_orientation(paths, labels, save_folder + "orientation.png", predicate) - evaluate_relativeOrientation( - paths, labels, save_folder + "relativeOrientation.png", predicate - ) - evaluate_distanceToWall( - paths, labels, save_folder + "distanceToWall.png", predicate - ) - evaluate_tankpositions(paths, labels, save_folder + "tankpositions.png", predicate) - evaluate_trajectories(paths, labels, save_folder + "trajectories.png", predicate) - evaluate_positionVec(paths, labels, save_folder + "posVec.png", predicate) - evaluate_follow_iid(paths, labels, save_folder + "follow_iid.png", predicate) + t = tqdm(fdict.items(), desc="Evaluation", leave=True) + for f_name, f_callable in t: + t.set_description(f_name) + t.refresh() # to show immediately the update + f_callable(paths, labels, save_folder / (f_name + ".png"), predicate) def calculate_follow(a, b): diff --git a/src/robofish/io/__init__.py b/src/robofish/io/__init__.py index 5e84ac01da5eccb45e7879027dc2589f851895e7..5b054c2fb71018dd23ceb315d83bf83f51cc1cac 100644 --- a/src/robofish/io/__init__.py +++ b/src/robofish/io/__init__.py @@ -1,5 +1,12 @@ # SPDX-License-Identifier: LGPL-3.0-or-later +""" +The Python package <a href="https://git.imp.fu-berlin.de/bioroboticslab/robofish/io">robofish.io</a> provides a simple interface to create, load, modify, and inspect files containing world information and movement tracks of entities (organisms, robots, obstacles,...). +The files are saved in the `.hdf5` format and following the [Track Format Specification](https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/uploads/f76d86e7a629ca38f472b8f23234dbb4/RoboFish_Track_Format_-_1.0.pdf). + +.. include:: ../../../docs/index.md +""" + import sys import logging diff --git a/src/robofish/io/app.py b/src/robofish/io/app.py index fc6fee0f27766b179fa494547b67cee736cd7083..e31e9ca38a8584312ec8250c6bb733a55b03c479 100644 --- a/src/robofish/io/app.py +++ b/src/robofish/io/app.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- +""" +.. include:: ../../../docs/app.md +""" + # ----------------------------------------------------------- -# Functions available to be used in the commandline to show and validate hdf5 -# according to the Robofish track format (1.0 Draft 7). The standard is -# available at https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format -# # Dec 2020 Andreas Gerken, Berlin, Germany # Released under GNU 3.0 License # email andi.gerken@gmail.com diff --git a/src/robofish/io/entity.py b/src/robofish/io/entity.py index e407989275322cb24fb20e62cef184e143166788..b3b7b219e52379b0d2f0355828538bb3c2d51202 100644 --- a/src/robofish/io/entity.py +++ b/src/robofish/io/entity.py @@ -1,3 +1,7 @@ +""" +.. include:: ../../../docs/entity.md +""" + import robofish.io import robofish.io.utils as utils @@ -134,31 +138,41 @@ class Entity(h5py.Group): def orientations(self): if not "orientations" in self: # If no orientation is given, the default direction is to the right - return np.tile([1, 0], (self.positions.shape[0], 1)) + return np.tile([0, 1], (self.positions.shape[0], 1)) return self["orientations"] @property - def orientations_calculated(self): - diff = np.diff(self.positions, axis=0) - angles = np.arctan2(diff[:, 1], diff[:, 0]) - return angles[:, np.newaxis] + def orientations_rad(self): + ori_rad = utils.limit_angle_range( + np.arctan2(self.orientations[:, 1], self.orientations[:, 0]), + _range=(0, 2 * np.pi), + ) + return ori_rad[:, np.newaxis] @property def poses_calc_ori_rad(self): - return np.concatenate( - [self.positions[:-1], self.orientations_calculated], axis=1 + # Diff between positions [t - 1, 2] + diff = np.diff(self.positions, axis=0) + + # angles [t - 1] + angles = utils.limit_angle_range( + np.arctan2(diff[:, 1], diff[:, 0]), _range=(0, 2 * np.pi) ) + # Positions with angles. The first position is cut of, as it does not have an orientation. + poses_with_calculated_orientation = np.concatenate( + [self.positions[1:], angles[:, np.newaxis]], axis=1 + ) + + return poses_with_calculated_orientation + @property def poses(self): return np.concatenate([self.positions, self.orientations], axis=1) @property def poses_rad(self): - poses = self.poses - # calculate the angles from the orientation vectors, write them to the third row and delete the fourth row - ori_rad = np.arctan2(poses[:, 3], poses[:, 2]) - return np.concatenate([poses[:, :2], ori_rad[:, np.newaxis]], axis=1) + return np.concatenate([self.positions, self.orientations_rad], axis=1) @property def speed_turn(self): @@ -173,10 +187,12 @@ class Entity(h5py.Group): We assume, that the entity is oriented "correctly" in the first pose. So the first turn angle is 0. """ - diff = np.diff(self.positions, axis=0) - speed = np.linalg.norm(diff, axis=1) - angles = np.arctan2(diff[:, 1], diff[:, 0]) - turn = np.zeros_like(angles) - turn[0] = 0 - turn[1:] = utils.limit_angle_range(np.diff(angles)) + # poses with calulated orientation have first position cut of as it does not have an orientation + # (t - 1, (x ,y, ori)) + poses_calc_ori = self.poses_calc_ori_rad + + # Differences cuts of last item (t - 2, (dx, dy, d ori)) + diff = np.diff(poses_calc_ori, axis=0) + speed = np.linalg.norm(diff[:, :2], axis=1) + turn = utils.limit_angle_range(diff[:, 2], _range=(-np.pi, np.pi)) return np.stack([speed, turn], axis=-1) diff --git a/src/robofish/io/file.py b/src/robofish/io/file.py index 9ff216c42b3bda4fd28c0aaade8d400d67254475..5d5742ffd3f28d63e038178db0ab296927e6249e 100644 --- a/src/robofish/io/file.py +++ b/src/robofish/io/file.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- +""" +.. include:: ../../../docs/file.md +""" + # ----------------------------------------------------------- # Utils functions for reading, validating and writing hdf5 files according to # Robofish track format (1.0 Draft 7). The standard is available at # https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format -# -# The term track is used to describe a dictionary, describing the track in a dict. -# To distinguish between attributes, dictionaries and groups, a prefix is used -# (a_ for attribute, d_ for dictionary, and g_ for groups). + # # Dec 2020 Andreas Gerken, Berlin, Germany # Released under GNU 3.0 License @@ -30,6 +31,7 @@ import uuid import deprecation import types +# Remember: Update docstring when updating these two global variables default_format_version = np.array([1, 0], dtype=np.int32) default_format_url = ( @@ -61,26 +63,64 @@ class File(h5py.File): ): """Create a new RoboFish Track Format object. - When called with a path, it is loaded, otherwise a new temporary file is created. + When called with a path, it is loaded, otherwise a new temporary + file is created. File contents can be validated against the + track format specification. Parameters ---------- path : str or Path, optional Location of file to be opened. If not provided, mode is ignored. - - mode : str - r Readonly, file must exist (default) - r+ Read/write, file must exist - w Create file, truncate if exists - x Create file, fail if exists - a Read/write if exists, create otherwise - - world_size_cm - optional integer array of the world size in cm - strict_validate - optional boolean, if the file should be strictly validated, when loaded from a path. The default is False. - format_version - optional version [major, minor] of the trackformat specification + mode : str, default='r' + 'r' Readonly, file must exist + 'r+' Read/write, file must exist + 'w' Create file, truncate if exists + 'x' Create file, fail if exists + 'a' Read/write if exists, create otherwise + world_size_cm : [int, int] , optional + side lengths [x, y] of the world in cm. + rectangular world shape is assumed. + TODO: Cuboid world is also possible in track format + strict_validate : bool, default=False + if the file should be strictly validated against the track + format specification, when loaded from a path. + TODO: Should this validate against the version sepcified in + format_version or just against the most recent version? + format_version : [int, int], default=[1,0] + version [major, minor] of track format specification + format_url : str, default="https://git.imp.fu-berlin.de/bioroboticslab/robofish/track_format/-/releases/1.0" + location of track format specification. + should fit `format_version`. + sampling_name : str, optional + How to specify your sampling: + + 1. (optional) + provide text description of your sampling in `sampling_name` + + 2.a (mandatory, if you have a constant sampling frequency) + specify `frequency_hz` with your sampling frequency in Hz + + 2.b (mandatory, if you do NOT have a constant sampling frequency) + specify `monotonic_time_points_us` with a list[1] of time + points in microseconds on a montonic clock, one for each + sample in your dataset. + + 3. (optional) + specify `calendar_time_points` with a list[2] of time points + in the ISO 8601 extended format with microsecond precision + and time zone designator[3], one for each sample in your + dataset. + + [1] any Iterable of int + [2] any Iterable of str + [3] example: "2020-11-18T13:21:34.117015+01:00" + + frequency_hz: int, optional + refer to explanation of `sampling_name` + monotonic_time_points_us: Iterable of int, optional + refer to explanation of `sampling_name` + calendar_time_points: Iterable of str, optional + refer to explanation of `sampling_name` """ if path is None: @@ -143,6 +183,8 @@ class File(h5py.File): Args: path: path to a io file as a string or path object. If no path is specified, the last known path (from loading or saving) is used. strict_validate: optional boolean, if the file should be strictly validated, before saving. The default is True. + Returns: + The file itself, so something like f = robofish.io.File().save_as("file.hdf5") works """ self.validate(strict_validate=strict_validate) @@ -155,6 +197,8 @@ class File(h5py.File): shutil.copyfile(Path(self.filename).resolve(), path) + return self + def create_sampling( self, name: str = None, @@ -190,10 +234,10 @@ class File(h5py.File): def format_calendar_time_point(p): if isinstance(p, datetime.datetime): assert p.tzinfo is not None, "Missing timezone for calendar point." - return p.isoformat(timespec="milliseconds") + return p.isoformat(timespec="microseconds") elif isinstance(p, str): assert p == datetime.datetime.fromisoformat(p).isoformat( - timespec="milliseconds" + timespec="microseconds" ) return p else: @@ -205,8 +249,6 @@ class File(h5py.File): format_calendar_time_point(p) for p in calendar_time_points ] - for c in calendar_time_points: - print(type(c)) sampling.create_dataset( "calendar_time_points", data=calendar_time_points, @@ -271,7 +313,7 @@ class File(h5py.File): Args: TODO - category: the of the entity. The canonical values are ['fish', 'robot', 'obstacle']. + category: the of the entity. The canonical values are ['organism', 'robot', 'obstacle']. poses: optional two dimensional array, containing the poses of the entity (x,y,orientation_x, orientation_y). poses_rad: optional two dimensional containing the poses of the entity (x,y, orientation_rad). name: optional name of the entity. If no name is given, the is used with an id (e.g. 'fish_1') @@ -309,7 +351,7 @@ class File(h5py.File): """Creates multiple entities. Args: - category: The common category for the entities. The canonical values are ['fish', 'robot', 'obstacle']. + category: The common category for the entities. The canonical values are ['organism', 'robot', 'obstacle']. poses: three dimensional array, containing the poses of the entity. name: optional array of names of the entities. If no names are given, the category is used with an id (e.g. 'fish_1') outlines: optional array, containing the outlines of the entities, either a three dimensional common outline array can be given, or a four dimensional array. @@ -321,7 +363,6 @@ class File(h5py.File): assert poses.ndim == 3 assert poses.shape[2] in [3, 4] agents = poses.shape[0] - timesteps = poses.shape[1] entity_names = [] for i in range(agents): @@ -357,9 +398,23 @@ class File(h5py.File): for name in self.entity_names ] + @property + def entity_positions(self): + return self.select_entity_property(None, entity_property=Entity.positions) + + @property + def entity_orientations(self): + return self.select_entity_property(None, entity_property=Entity.orientations) + + @property + def entity_orientations_rad(self): + return self.select_entity_property( + None, entity_property=Entity.orientations_rad + ) + @property def entity_poses(self): - return self.select_entity_property(None) + return self.select_entity_property(None, entity_property=Entity.poses) @property def entity_poses_rad(self): diff --git a/src/robofish/io/io.py b/src/robofish/io/io.py index c8894ca38365340509b94e59b5ea85374357109d..4c12aded73ffab9675370b160dac54b832164331 100644 --- a/src/robofish/io/io.py +++ b/src/robofish/io/io.py @@ -16,7 +16,7 @@ def now_iso8061() -> str: str: The current time as iso8061 string. """ return datetime.datetime.now(datetime.timezone.utc).isoformat( - timespec="milliseconds" + timespec="microseconds" ) diff --git a/src/robofish/io/validation.py b/src/robofish/io/validation.py index f8ae264c83f8314deab290db71bd2318da939f71..c5ac4a68766ea5dd8ff66cd2ebf087d42879818a 100644 --- a/src/robofish/io/validation.py +++ b/src/robofish/io/validation.py @@ -113,9 +113,7 @@ def validate(iofile: File, strict_validate: bool = True) -> (bool, str): if a in sampling: assert_validate_type(sampling[a], a_type, a, f"sampling {s_name}") - if "frequency_hz" in sampling.attrs: - pass - elif "monotonic_time_points_us" in sampling: + if "monotonic_time_points_us" in sampling: time_points = sampling["monotonic_time_points_us"] # 1 dimensional array assert_validate( @@ -138,12 +136,13 @@ def validate(iofile: File, strict_validate: bool = True) -> (bool, str): "Dimensionality of calendar_time_points should be 1", f"sampling {s_name}", ) - assert_validate( - calendar_points.shape[0] == time_points.shape[0], - "The length of calendar points (%d) does not match the length of monotonic points (%d)" - % (calendar_points.shape[0], time_points.shape[0]), - f"sampling {s_name}", - ) + if "monotonic_time_points_us" in sampling: + assert_validate( + calendar_points.shape[0] == time_points.shape[0], + "The length of calendar points (%d) does not match the length of monotonic points (%d)" + % (calendar_points.shape[0], time_points.shape[0]), + f"sampling {s_name}", + ) # validate iso8601, this validates the dtype implicitly for c in calendar_points.asstr(encoding="utf-8"): diff --git a/tests/robofish/evaluate/test_app_evaluate.py b/tests/robofish/evaluate/test_app_evaluate.py index fce024a3c4f7e030b82e301bd9c8871b675beca7..d95fa70893f8866920844c0a5564ed9693c8a9f6 100644 --- a/tests/robofish/evaluate/test_app_evaluate.py +++ b/tests/robofish/evaluate/test_app_evaluate.py @@ -23,5 +23,6 @@ def test_app_validate(): self.save_path = graphics_out for mode in app.function_dict().keys(): - app.evaluate(DummyArgs(mode)) + if mode != "all": + app.evaluate(DummyArgs(mode)) graphics_out.unlink() diff --git a/tests/robofish/io/test_entity.py b/tests/robofish/io/test_entity.py index 7427f5aa4848715a8650b905fdfd0e1866712d14..72dc5f7efe8a0ed1296a66f39473e575230dd123 100644 --- a/tests/robofish/io/test_entity.py +++ b/tests/robofish/io/test_entity.py @@ -6,9 +6,9 @@ import numpy as np def test_entity_object(): sf = robofish.io.File(world_size_cm=[100, 100], frequency_hz=25) f = sf.create_entity("fish", positions=[[10, 10]]) - assert type(f) == robofish.io.Entity - assert f.name == "fish_1" - assert f.attrs["category"] == "fish" + assert type(f) == robofish.io.Entity, "Type of entity was wrong" + assert f.name == "fish_1", "Name of entity was wrong" + assert f.attrs["category"] == "fish", "category was wrong" print(dir(f)) print(f["positions"]) assert type(f["positions"]) == h5py.Dataset @@ -40,13 +40,13 @@ def test_entity_turn_speed(): ) e = f.create_entity("fish", positions=positions) speed_turn = e.speed_turn - assert speed_turn.shape == (99, 2) + assert speed_turn.shape == (98, 2) - # No turn in the first timestep, since initialization turns it the right way - assert speed_turn[0, 1] == 0 + # Turns and speeds shoud be all the same afterwards, since the fish swims with constant velocity and angular velocity. + assert (np.std(speed_turn, axis=0) < 0.0001).all() - # Turns and speeds shoud afterwards be all the same afterwards, since the fish swims with constant velocity and angular velocity. - assert (np.std(speed_turn[1:], axis=0) < 0.0001).all() + # Cut off the first position as it cannot be simulated + positions = positions[1:] # Use turn_speed to generate positions gen_positions = np.zeros((positions.shape[0], 3)) diff --git a/tests/robofish/io/test_file.py b/tests/robofish/io/test_file.py index fe6f1f9c5f8dbbcb88a3a9162aaa324da3c8df2e..31b3bda15fd51f3ba87b988fa488ff8c2f218f53 100644 --- a/tests/robofish/io/test_file.py +++ b/tests/robofish/io/test_file.py @@ -112,7 +112,7 @@ def test_multiple_entities(): # create new sampling m_points = np.ones((timesteps)) c_points = np.empty((timesteps), dtype="O") - c_points[:5] = "2020-12-02T10:21:58.100+00:00" + c_points[:5] = "2020-12-02T10:21:58.100000+00:00" c_points[5:] = robofish.io.now_iso8061() new_sampling = sf.create_sampling( @@ -184,8 +184,10 @@ def test_entity_positions_no_orientation(): # Create an entity, using radiants f.create_entity("fish", positions=np.ones((100, 2))) + # In poses, the default orientation pointing up should be added. assert f.entity_poses.shape == (1, 100, 4) - assert (f.entity_poses[:, :] == np.array([1, 1, 1, 0])).all() + assert (f.entity_poses[:, :] == np.array([1, 1, 0, 1])).all() + assert np.isclose(f.entity_orientations_rad, np.pi / 2).all() # Calculate the orientation assert f.entity_poses_calc_ori_rad.shape == (1, 99, 3) diff --git a/tests/robofish/io/test_io.py b/tests/robofish/io/test_io.py index bb4cb69365d8bc8d325fb1bf7cd502fb0be536f3..3a7bb32390223adfbd4e7f2f1f9a3454c93bdfa4 100644 --- a/tests/robofish/io/test_io.py +++ b/tests/robofish/io/test_io.py @@ -6,10 +6,10 @@ import numpy as np def test_now_iso8061(): - # Example time: 2021-01-05T14:33:40.401+00:00 + # Example time: 2021-01-05T14:33:40.401000+00:00 time = robofish.io.now_iso8061() assert type(time) == str - assert len(time) == 29 + assert len(time) == 32 def test_read_multiple_single():