Skip to content
Snippets Groups Projects
Commit e14bad89 authored by Andi Gerken's avatar Andi Gerken
Browse files

Added hash, create tracks,...

parent 2c9cbaf2
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 479 additions and 28 deletions
automation/calculate_model_hashes.py 0 → 100644
+ 19
− 0
View file @ e14bad89
from pathlib import Path
import fish_models
from pprint import pprint
import json
model_path = Path("../public_html/models")
model_hash_json = Path("../public_html/model_hashes.json")
result_json = {}
for model in model_path.glob("*.zip"):
sha1 = fish_models.utils.file_hash(model)
result_json[model.name] = sha1
with open(model_hash_json, "w") as mh_json:
json.dump(result_json, mh_json)
pprint(result_json)
print(f"Hashes stored in {str(model_hash_json)}")
automation/create_tracks.ipynb
+ 1
− 1
View file @ e14bad89
......@@ -165,7 +165,7 @@
"hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
},
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
public_html/index.php
+ 10
− 0
View file @ e14bad89
......@@ -43,6 +43,7 @@
<th>Size</th>
<th>Generated Tracks</th>
<th>Evaluation</th>
<th>Video</th>
<th>Model</th>
</tr>
<?php
......@@ -71,6 +72,11 @@
}else{
echo"<td></td>";
}
if(file_exists($evaluation_path."/track_0.mp4")){
echo "<td><a href=".$evaluation_path."/track_0.mp4>Video</a></td>";
}else{
echo "<td></td>";
}
if(file_exists($model_args['path'])){
echo "<td><a href='", $model_args['path'], "'>Download</a></td>";
}
......@@ -85,6 +91,10 @@
?>
</table>
<div>
<a href = "model_hashes.json">Model hashes (sha1)</a>
</div>
<div>
<h2>Models without training</h2>
Models which don't require training can be generated using the same <code>fish_models.ModelStorage.load_from_model_name("model.class.path(arg=42)")</code> function with the class path and arguments passed as a string (from version 0.3.7).
......
public_html/model_details.php
+ 7
− 1
View file @ e14bad89
......@@ -17,6 +17,12 @@
$folder = "evaluation/models/".$model;
if (is_dir($folder)){
$videos = glob($folder."/*.{mp4}", GLOB_BRACE);
foreach($videos as $video){
echo "<h4>".end((explode('/', $file)))."</h2>";
echo "<video width='640' height='640' controls><source src=$video type='video/mp4'></video>";
}
$files = glob($folder."/*.{png,jpg}", GLOB_BRACE);
foreach($files as $file) {
echo "<h4>".end((explode('/', $file)))."</h2>";
......
public_html/model_hashes.json 0 → 100644
+ 1
− 0
View file @ e14bad89
{"ReplayModel_v0.zip": "2e131cadbf30d78156a5ba4d0e2af0aeafdd6bc3", "GRU_with_conv_15fov.zip": "1be32ccdf9f28d428e11724d8e6f61e6fb1b16af", "KNNModel_v0.zip": "9ffa06b66a1519b7a44b325baddb5ed12e79ac33", "GRU_with_conv_51fov.zip": "f8f4d3729b4c94dfd8ff176802df7d7c1e142d4e", "GRU_without_conv_51_fov.zip": "c16a383de8dca96b5f4758fc66c70535985a0f15", "ClusterModel_v1_0.zip": "6148edf3820ba1a3227b192bd15a0f83d2c26d46", "ClassificationModel_v0.zip": "8945a0156dec3bc2a7269060c5b0a51d0525f0db", "ConvLinearDropoutClassifier_v0.zip": "8fc3561a93774f9187ae33ce9ed8ee226943533f", "ConvLinearDropoutClassifier_v1_trial_399.zip": "ec8fabecef782fc2e0f741dba7e0e2b9e711b2da", "ConvLinearDropoutClassifier_v1_trial_405.zip": "e283c494115e815ec8f5f064070bf0087654212b", "ClusterModel_v1_1.zip": "2de7e594c695b558644e19abd56d279bfb093fce", "ClusterModel_v0.zip": "52d184745f1f7879f7244845588376c314ef8b4a"}
\ No newline at end of file
public_html/models.json
+ 70
− 24
View file @ e14bad89
......@@ -17,7 +17,10 @@
},
"train_args": {},
"dataset": "lfftr_av_6w_6f_rd2_mf5",
"evaluation_folders": ["live_female_female/train", "live_female_female/test"]
"evaluation_folders": [
"live_female_female/train",
"live_female_female/test"
]
},
"ClusterModel_v0": {
"path": "models/ClusterModel_v0.zip",
......@@ -31,26 +34,69 @@
}
},
"dataset": "lfftr_av_4_2piw_10_2pif_rd2_mf20",
"evaluation_folders": ["live_female_female/train", "live_female_female/test"]
"evaluation_folders": [
"live_female_female/train",
"live_female_female/test"
]
},
"ClusterModel_v1_0": {
"path": "models/ClusterModel_v1_0.zip",
"date": "29.11.2021",
"notes": "This model is time dependent. This is currently the best working model.",
"dataset": "Full cherry_train dataset",
"evaluation_folders": ["live_female_female/cherry_train", "live_female_female/cherry_test"]
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
},
"ClusterModel_v1_1": {
"path": "models/ClusterModel_v1_1.zip",
"date": "29.11.2021",
"notes": "This model is time dependent. This comes directly from hyperparameter optimization.",
"dataset": "4 cut files of cherry_train dataset",
"evaluation_folders": ["live_female_female/cherry_train", "live_female_female/cherry_test"]
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
},
"LiveFemaleFemaleConstantSpeedCouzinModel()": {
"class": "fish_models.models.released.parametrized.LiveFemaleFemaleConstantSpeedCouzinModel()",
"date": "27.01.2021",
"notes": "Parametrized, for details see below.",
"dataset": "cherry_train",
"evaluation_folders": ["live_female_female/cherry_train", "live_female_female/cherry_test"]
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
},
"ConvLinearDropoutClassifier_v0": {
"path": "models/ConvLinearDropoutClassifier_v0.zip",
"date": "14.04.2022",
"notes": "This model is time dependent, it uses 2 conv layers and two linear layers.",
"dataset": "cherry_train",
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
},
"ConvLinearDropoutClassifier_v1_trial_399": {
"path": "models/ConvLinearDropoutClassifier_v1_trial_399.zip",
"date": "20.04.2022",
"notes": "A candidate for the best hyperparameter trial when using wasserstein evaluation with <code>['speed', 'turn', 'iid', 'wall_dists', 'follow', 'tlvc']</code> and their first derivation. <a href='resources/hyper_trial_399.yaml'>Hyperparameters</a>",
"dataset": "cherry_train",
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
},
"ConvLinearDropoutClassifier_v1_trial_405": {
"path": "models/ConvLinearDropoutClassifier_v1_trial_405.zip",
"date": "20.04.2022",
"notes": "A candidate for the best hyperparameter trial when using wasserstein evaluation with <code>['speed', 'turn', 'iid', 'wall_dists']</code> and their first two derivations. <a href='resources/hyper_trial_405.yaml'>Hyperparameters</a>",
"dataset": "cherry_train",
"evaluation_folders": [
"live_female_female/cherry_train",
"live_female_female/cherry_test"
]
}
}
\ No newline at end of file
public_html/raw_data.json
+ 15
− 2
View file @ e14bad89
......@@ -32,7 +32,7 @@
"author": "Pascal Mueller",
"date": "12.10.2021",
"tracking": "Simulation",
"notes": "The couzin guppies have a 180 degree FOV.<br>10.000 files which take long to load. Poses and actions are available in the dataset with the same name.",
"notes": "Smooth couzin guppies with a 180 degree FOV.<br>ZOR: 10cm, ZOO: 20cm, ZOA: 40cm, ZOWR: 15cm, initial_poses minimum_wall_distance: 15cm.",
"folders": [
"train",
"test",
......@@ -45,7 +45,20 @@
"author": "Pascal Mueller",
"date": "12.10.2021",
"tracking": "Simulation",
"notes": "The couzin guppies have a 360 degree FOV.<br>10.000 files which take long to load. Poses and actions are available in the dataset with the same name.",
"notes": "2 smooth couzin guppies with a 360 degree FOV.<br>ZOR: 10cm, ZOO: 20cm, ZOA: 40cm, ZOWR: 15cm, initial_poses minimum_wall_distance: 15cm.",
"folders": [
"train",
"test",
"validation"
]
},
"pascal_6fish_couzin": {
"path": "raw_data/pascal_6fish_couzin.zip",
"path_unpacked": "raw_data/pascal_6fish_couzin",
"author": "Andi Gerken",
"date": "01.04.2022",
"tracking": "Simulation",
"notes": "6 smooth couzin guppies with a 360 degree FOV.<br>ZOR: 10cm, ZOO: 20cm, ZOA: 40cm, ZOWR: 15cm, initial_poses minimum_wall_distance: 15cm.",
"folders": [
"train",
"test",
......
public_html/resources/hyper_trial_399.yaml 0 → 100644
+ 178
− 0
View file @ e14bad89
attributed_poses_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/grid_poses/attributed
base_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd
constant_speed: null
data_folder_test: /home/andi/blubber_workspace/fish_models/storage/raw_data/live_female_female/cherry_test
data_folder_train: /home/andi/blubber_workspace/fish_models/storage/raw_data/live_female_female/cherry_train
data_name: live_female_female
dataset_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets
device: cuda
dset_generated_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/datasets/generated_dset_trial_399.pkl
dset_test_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/test_dset.pkl
dset_train_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/train_dset.pkl
e_stack_frames: 1.5240880437090876
epochs: 50
evaluation_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/evaluation
fov_angle_fish_bins: 6.283185307
fov_angle_wall_raycasts: 6.283185307
generated_tracks_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/generated_tracks/trial_399
grid_dset_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/grid_poses.pkl
grid_poses_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/grid_poses/raw
grid_side_length: 500
hyperopt_config_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/configs/live_full_hyperparams_d_dd_trial_%d.yaml
hyperopt_dset_generated_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/datasets/generated_dset_trial_%d.pkl
hyperopt_generated_tracks_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/generated_tracks/trial_%d
hyperopt_model_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/models/live_full_hyperparams_d_dd_hyper_trial_%d.pkl
hyperopt_params:
e_stack_frames:
- float
- 1
- 2
learning_rate:
- float
- 0.0001
- 0.01
model_options:
conv_layer0:
- int
- 1
- 300
conv_layer1:
- int
- 1
- 300
conv_n_layers:
- int
- 1
- 2
hidden_layer0:
- int
- 1
- 500
hidden_layer1:
- int
- 1
- 500
hidden_layers:
- int
- 1
- 2
p_dropout0:
- float
- 0
- 0.5
p_dropout1:
- float
- 0
- 0.5
n_fish_bins:
- int
- 5
- 50
n_stack_frames:
- int
- 1
- 10
n_wall_raycasts:
- int
- 5
- 50
speed_bins:
- int
- 5
- 250
turn_bins:
- int
- 5
- 250
hyperopt_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt
hyperopt_storage: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/database/hyperopt.db
hyperopt_trials: 200
hyperopt_wasserstein_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/wasserstein/wasserstein_trial_%d.json
learning_rate: 0.0003921821606683175
loss_fn: SplitLoss
max_files_test: null
max_files_train: null
model_class: fish_models.models.andi.pascal_like_models.MLPClassifier
model_n_track_factor: 20
model_options:
conv_layers_sizes:
- 151
- 182
hidden_layers:
- 357
- 455
p_dropout:
- 0.18494407716438724
- 0.06285413243283158
reduce_stacked_dimensions: false
stack_frames: &id002
- !!python/object/apply:numpy.core.multiarray.scalar
- &id001 !!python/object/apply:numpy.dtype
args:
- i8
- false
- true
state: !!python/tuple
- 3
- <
- null
- null
- null
- -1
- -1
- 0
- !!binary |
/v////////8=
- !!python/object/apply:numpy.core.multiarray.scalar
- *id001
- !!binary |
//////////8=
model_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/models/live_full_hyperparams_d_dd_hyper_trial_399.pkl
n_fish_bins: 8
n_stack_frames: 2
n_wall_raycasts: 38
on_n_repeats: 10
on_n_timesteps: 100
on_n_tracks: 15
print_every: 1
reduce_stacked_dimensions: false
required_keys:
- training_name
- data_name
- data_folder_train
- data_folder_test
- max_files_train
- max_files_test
- constant_speed
- speed_bins
- turn_bins
- stack_frames
- reduce_stacked_dimensions
- model_options
- model_class
- learning_rate
- loss_fn
- epochs
- print_every
- device
- on_n_tracks
- on_n_repeats
- on_n_timesteps
- hyperopt_storage
- hyperopt_trials
- grid_side_length
- base_path
- dset_train_path
- dset_test_path
- model_path
- grid_poses_path
- grid_dset_path
- attributed_poses_path
- generated_tracks_path
- dset_generated_path
- evaluation_path
- hyperopt_config_path
speed_bins: 173
stack_frames: *id002
training_name: live_full_hyperparams_d_dd
turn_bins: 69
public_html/resources/hyper_trial_405.yaml 0 → 100644
+ 178
− 0
View file @ e14bad89
attributed_poses_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/grid_poses/attributed
base_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd
constant_speed: null
data_folder_test: /home/andi/blubber_workspace/fish_models/storage/raw_data/live_female_female/cherry_test
data_folder_train: /home/andi/blubber_workspace/fish_models/storage/raw_data/live_female_female/cherry_train
data_name: live_female_female
dataset_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets
device: cuda
dset_generated_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/datasets/generated_dset_trial_405.pkl
dset_test_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/test_dset.pkl
dset_train_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/train_dset.pkl
e_stack_frames: 1.4989516291433655
epochs: 50
evaluation_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/evaluation
fov_angle_fish_bins: 6.283185307
fov_angle_wall_raycasts: 6.283185307
generated_tracks_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/generated_tracks/trial_405
grid_dset_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/datasets/grid_poses.pkl
grid_poses_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/grid_poses/raw
grid_side_length: 500
hyperopt_config_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/configs/live_full_hyperparams_d_dd_trial_%d.yaml
hyperopt_dset_generated_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/datasets/generated_dset_trial_%d.pkl
hyperopt_generated_tracks_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/generated_tracks/trial_%d
hyperopt_model_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/models/live_full_hyperparams_d_dd_hyper_trial_%d.pkl
hyperopt_params:
e_stack_frames:
- float
- 1
- 2
learning_rate:
- float
- 0.0001
- 0.01
model_options:
conv_layer0:
- int
- 1
- 300
conv_layer1:
- int
- 1
- 300
conv_n_layers:
- int
- 1
- 2
hidden_layer0:
- int
- 1
- 500
hidden_layer1:
- int
- 1
- 500
hidden_layers:
- int
- 1
- 2
p_dropout0:
- float
- 0
- 0.5
p_dropout1:
- float
- 0
- 0.5
n_fish_bins:
- int
- 5
- 50
n_stack_frames:
- int
- 1
- 10
n_wall_raycasts:
- int
- 5
- 50
speed_bins:
- int
- 5
- 250
turn_bins:
- int
- 5
- 250
hyperopt_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt
hyperopt_storage: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/database/hyperopt.db
hyperopt_trials: 200
hyperopt_wasserstein_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/wasserstein/wasserstein_trial_%d.json
learning_rate: 0.00045698280190765853
loss_fn: SplitLoss
max_files_test: null
max_files_train: null
model_class: fish_models.models.andi.pascal_like_models.MLPClassifier
model_n_track_factor: 20
model_options:
conv_layers_sizes:
- 160
- 181
hidden_layers:
- 352
- 441
p_dropout:
- 0.19517537082038344
- 0.0694873954903051
reduce_stacked_dimensions: false
stack_frames: &id002
- !!python/object/apply:numpy.core.multiarray.scalar
- &id001 !!python/object/apply:numpy.dtype
args:
- i8
- false
- true
state: !!python/tuple
- 3
- <
- null
- null
- null
- -1
- -1
- 0
- !!binary |
/v////////8=
- !!python/object/apply:numpy.core.multiarray.scalar
- *id001
- !!binary |
//////////8=
model_path: /home/andi/blubber_workspace/fish_models/experiments/training_pipeline/results/live_full_hyperparams_d_dd/hyperopt/models/live_full_hyperparams_d_dd_hyper_trial_405.pkl
n_fish_bins: 5
n_stack_frames: 2
n_wall_raycasts: 37
on_n_repeats: 10
on_n_timesteps: 100
on_n_tracks: 15
print_every: 1
reduce_stacked_dimensions: false
required_keys:
- training_name
- data_name
- data_folder_train
- data_folder_test
- max_files_train
- max_files_test
- constant_speed
- speed_bins
- turn_bins
- stack_frames
- reduce_stacked_dimensions
- model_options
- model_class
- learning_rate
- loss_fn
- epochs
- print_every
- device
- on_n_tracks
- on_n_repeats
- on_n_timesteps
- hyperopt_storage
- hyperopt_trials
- grid_side_length
- base_path
- dset_train_path
- dset_test_path
- model_path
- grid_poses_path
- grid_dset_path
- attributed_poses_path
- generated_tracks_path
- dset_generated_path
- evaluation_path
- hyperopt_config_path
speed_bins: 160
stack_frames: *id002
training_name: live_full_hyperparams_d_dd
turn_bins: 76
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment