Skip to content
Snippets Groups Projects
Commit 2d53557c authored by Andi Gerken's avatar Andi Gerken
Browse files

Initial commit

parents
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 629 additions and 0 deletions
automation/_create_dset.ipynb 0 → 100644
+ 129
− 0
View file @ 2d53557c
%% Cell type:code id: tags:
```
%reload_ext autoreload
%autoreload 2
import fish_models
import numpy as np
import copy
from pathlib import Path
import json
requested_dset = "lfftr_aav_4_2piw_10_2pif_rd2_mf10_oris"
dataset_json = json.load(open("../buffer/datasets.json", "r"))
meta_config = dataset_json[requested_dset]
config = meta_config["dataset_config"]
```
%% Cell type:code id: tags:
```
modified_config = copy.deepcopy(config)
if config["data_path"] == "live_female_female/train":
modified_config["data_path"] = fish_models.live_female_female_data() / "train"
else:
raise Exception("Not supported data")
translate = {
"pi":np.pi,
"2pi":2 * np.pi
}
for k, v in modified_config["raycast"].items():
if type(v) is not list and v in translate:
modified_config["raycast"][k] = translate[v]
if "speed_bins" in modified_config:
sb = modified_config["speed_bins"]
modified_config["speed_bin_borders"] = np.linspace(sb[0], sb[1], sb[2])
modified_config.pop("speed_bins")
if "turn_bins" in modified_config:
sb = modified_config["turn_bins"]
modified_config["turn_bin_borders"] = np.linspace(sb[0], sb[1], sb[2])
modified_config.pop("turn_bins")
raycast = fish_models.gym_interface.Raycast(world_bounds=([-50, -50], [50, 50]),**modified_config["raycast"])
modified_config["raycast"] = raycast
dset = fish_models.datasets.io_dataset.IoDataset(**modified_config)
storage_path = Path("../buffer") / meta_config["path"]
if storage_path.exists():
raise Exception(f"File exists and will be overwritten.")
dset.store(storage_path)
```
%% Output
Loading data from 20 files.
100%|██████████| 20/20 [00:23<00:00, 1.17s/it]
Calculating views from 20 files.
20%|██ | 4/20 [00:11<00:48, 3.01s/it]
%% Cell type:code id: tags:
```
```
automation/commands.md 0 → 100644
+ 4
− 0
View file @ 2d53557c
# Correct settings for php server
find . -type d -exec chmod 755 {} \;
find . -type f -exec chmod 655 {} \;
automation/config_new_dset.ipynb 0 → 100644
+ 113
− 0
View file @ 2d53557c
%% Cell type:code id: tags:
```
import datetime
import json
dataset_config= {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 4,
"n_fish_bins": 10,
"fov_angle_fish_bins": "2pi",
"fov_angle_wall_raycasts": "2pi"
},
"output_strings": [
"actions",
"actions_binned",
"views"
],
"speed_bins":(-2,20,21),
"turn_bins":(-10,10,21),
"reduce_dim": 2,
"max_files": 10
}
name = ""
if dataset_config["data_path"] == "live_female_female/train":
name += "lfftr_"
for o in dataset_config["output_strings"]:
# add the first character of the output strings
name += o[0]
ray_config = dataset_config["raycast"]
name += f"_{ray_config['n_wall_raycasts']}_{ray_config['fov_angle_wall_raycasts']}w_{ray_config['n_fish_bins']}_{ray_config['fov_angle_fish_bins']}f_rd{dataset_config['reduce_dim']}"
if dataset_config["max_files"] is not None:
name += f"_mf{dataset_config['max_files']}"
today = datetime.date.today().strftime("%d.%m.%Y")
output_config = {name:{"path":f"datasets/{name + '.pickle'}", "date":today, "dataset_config":dataset_config}}
print(json.dumps(output_config, indent=4))
```
%% Output
{
"lfftr_av_4_2piw_10_2pif_rd2_mf20": {
"path": "datasets/lfftr_av_4_2piw_10_2pif_rd2_mf20.pickle",
"date": "01.09.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 4,
"n_fish_bins": 10,
"fov_angle_fish_bins": "2pi",
"fov_angle_wall_raycasts": "2pi"
},
"output_strings": [
"actions",
"views"
],
"reduce_dim": 2,
"max_files": 20
}
}
}
%% Cell type:code id: tags:
```
```
automation/create_dset.ipynb 0 → 100644
+ 183
− 0
View file @ 2d53557c
%% Cell type:code id: tags:
```
%reload_ext autoreload
%autoreload 2
import fish_models
import numpy as np
import copy
from pathlib import Path
import json
requested_dset = "lfftr_aav_4_2piw_10_2pif_rd2_mf10_oris"
dataset_json = json.load(open("../buffer/datasets.json", "r"))
meta_config = dataset_json[requested_dset]
config = meta_config["dataset_config"]
```
%% Cell type:code id: tags:
```
modified_config = copy.deepcopy(config)
if config["data_path"] == "live_female_female/train":
modified_config["data_path"] = fish_models.live_female_female_data() / "train"
else:
raise Exception("Not supported data")
translate = {
"pi":np.pi,
"2pi":2 * np.pi
}
for k, v in modified_config["raycast"].items():
if type(v) is not list and v in translate:
modified_config["raycast"][k] = translate[v]
if "speed_bins" in modified_config:
sb = modified_config["speed_bins"]
modified_config["speed_bin_borders"] = np.linspace(sb[0], sb[1], sb[2])
modified_config.pop("speed_bins")
if "turn_bins" in modified_config:
sb = modified_config["turn_bins"]
modified_config["turn_bin_borders"] = np.linspace(sb[0], sb[1], sb[2])
modified_config.pop("turn_bins")
raycast = fish_models.gym_interface.Raycast(world_bounds=([-50, -50], [50, 50]),**modified_config["raycast"])
modified_config["raycast"] = raycast
dset = fish_models.datasets.io_dataset.IoDataset(**modified_config)
storage_path = Path("../buffer") / meta_config["path"]
if storage_path.exists():
raise Exception(f"File exists and will be overwritten.")
dset.store(storage_path)
```
%% Output
Loading data from 20 files.
100%|██████████| 20/20 [00:23<00:00, 1.17s/it]
Calculating views from 20 files.
100%|██████████| 20/20 [01:00<00:00, 3.02s/it]
Reducing shape of actions.
100%|██████████| 359560/359560 [00:02<00:00, 147861.50it/s]
Reducing shape of actions_binned.
100%|██████████| 359560/359560 [00:02<00:00, 156212.84it/s]
Reducing shape of views.
100%|██████████| 359560/359560 [00:02<00:00, 153981.95it/s]
Status of IoDataset:
The first 3 dimensions are reduced from (20, 2, 8989) to (359560)
actions (359560, 2): consisting of speed [cm/s] and turn [rad/s].
actions_binned (359560, 2): consisting of speed [discretized into 20 bins] and turn [discretized into 20 bins]
views (359560, 24): consisting of 10 fish bins, 4 wall raycasts and 10 fish oris.
%% Cell type:code id: tags:
```
```
automation/create_model.ipynb 0 → 100644
+ 0
− 0
View file @ 2d53557c
This diff is collapsed.
automation/create_tracks.ipynb 0 → 100644
+ 0
− 0
View file @ 2d53557c
This diff is collapsed.
automation/model_server_watch.sh 0 → 100755
+ 7
− 0
View file @ 2d53557c
inotifywait -m ~/blubber_workspace/model_server/public_html/ -e create -e moved_to -e modify -e attrib -e close_write -e delete |
while read dir action file; do
if [[ ! "$file" =~ ".+\.swp" ]]; then
echo "$action happened to $file, copying everything"
rsync -a --delete /home/andi/blubber_workspace/model_server/public_html/ login_zedat:public_html/model_server/
fi
done
buffer/datasets.json 0 → 100644
+ 126
− 0
View file @ 2d53557c
{
"lfftr_av_6w_6f_rd2_mf5": {
"path": "datasets/lfftr_av_6w_6f_rd2_mf5.pickle",
"date": "31.08.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 6,
"n_fish_bins": 6,
"fov_angle_fish_bins": "pi",
"fov_angle_wall_raycasts": "pi"
},
"output_strings": [
"actions",
"views"
],
"reduce_dim": 2,
"max_files": 5
}
},
"lfftr_pav_10w_10f_rd2": {
"path": "datasets/lfftr_pav_10w_10f_rd2.pickle",
"date": "31.08.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 10,
"n_fish_bins": 10,
"fov_angle_fish_bins": "pi",
"fov_angle_wall_raycasts": "pi"
},
"output_strings": [
"poses",
"actions",
"views"
],
"reduce_dim": 2,
"max_files": null
}
},
"lfftr_av_4_2piw_10_2pif_rd2_mf20": {
"path": "datasets/lfftr_av_4_2piw_10_2pif_rd2_mf20.pickle",
"date": "01.09.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 4,
"n_fish_bins": 10,
"fov_angle_fish_bins": "2pi",
"fov_angle_wall_raycasts": "2pi"
},
"output_strings": [
"actions",
"views"
],
"reduce_dim": 2,
"max_files": 20
}
},
"lfftr_aav_4_2piw_10_2pif_rd2_mf10": {
"path": "datasets/lfftr_aav_4_2piw_10_2pif_rd2_mf10.pickle",
"date": "05.10.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 4,
"n_fish_bins": 10,
"fov_angle_fish_bins": "2pi",
"fov_angle_wall_raycasts": "2pi"
},
"output_strings": [
"actions",
"actions_binned",
"views"
],
"speed_bins": [
-2,
20,
21
],
"turn_bins": [
-10,
10,
21
],
"reduce_dim": 2,
"max_files": 20
}
},
"lfftr_aav_4_2piw_10_2pif_rd2_mf10_oris": {
"path": "datasets/lfftr_aav_4_2piw_10_2pif_rd2_mf10.pickle",
"date": "05.10.2021",
"dataset_config": {
"data_path": "live_female_female/train",
"raycast": {
"n_wall_raycasts": 4,
"n_fish_bins": 10,
"fov_angle_fish_bins": "2pi",
"fov_angle_wall_raycasts": "2pi",
"view_of": [
"fish",
"walls",
"fish_oris"
]
},
"output_strings": [
"actions",
"actions_binned",
"views"
],
"speed_bins": [
-2,
20,
21
],
"turn_bins": [
-10,
10,
21
],
"reduce_dim": 2,
"max_files": 20
}
}
}
\ No newline at end of file
buffer/models.json 0 → 100644
+ 67
− 0
View file @ 2d53557c
{
"ReplayModel_v0": {
"path": "models/ReplayModel_v0.zip",
"date": "31.08.2021",
"model_class": "fish_models.models.released.replay_model.ReplayModel",
"model_args": {},
"train_args": {},
"dataset": "lfftr_av_6w_6f_rd2_mf5"
},
"KNNModel_v0": {
"path": "models/KNNModel_v0.zip",
"date": "31.08.2021",
"model_class": "fish_models.models.released.knn_model.KNNModel",
"model_args": {
"k": 3
},
"train_args": {},
"dataset": "lfftr_av_6w_6f_rd2_mf5"
},
"ClusterModel_v0": {
"path": "models/ClusterModel_v0.zip",
"date": "01.09.2021",
"model_class": "fish_models.models.released.cluster_model.ClusterModel",
"model_args": {
"cluster_method": "KMeans",
"cluster_args": {
"n_init": 100,
"n_clusters": 200
}
},
"train_args": {},
"dataset": "lfftr_av_4_2piw_10_2pif_rd2_mf20"
},
"ClusterModel_v1": {
"path": "models/ClusterModel_v1.zip",
"date": "01.09.2021",
"model_class": "fish_models.models.released.cluster_model.ClusterModel",
"model_args": {
"cluster_method": "KMeans",
"cluster_args": {
"n_init": 100,
"n_clusters": 200
}
},
"description": "This model was trained with fish, wall and fish_oris inputs.",
"train_args": {},
"dataset": "lfftr_aav_4_2piw_10_2pif_rd2_mf10"
},
"ClassificationModel_v0": {
"path": "models/ClassificationModel_v0.zip",
"date": "05.10.2021",
"model_class": "fish_models.models.andi.classification_model.ClassificationModel",
"model_args": {
"classifier_method": "MLP",
"classifier_args": {
"verbose": true,
"max_iter": 30,
"hidden_layer_sizes": [
128
]
}
},
"train_args": {},
"dataset": "lfftr_aav_4_2piw_10_2pif_rd2_mf10"
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment