Skip to content
Snippets Groups Projects
Commit bfd7abd0 authored by Max Breitenfeldt's avatar Max Breitenfeldt
Browse files

Add Python communication

parent f5e11f9a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitignore
+ 3
− 1
View file @ bfd7abd0
......@@ -75,4 +75,6 @@ bst_workspace.code-workspace
*.ini
*.xml
*.csv
build
\ No newline at end of file
build/**
.ccls-cache/**
CMakeLists.txt
+ 1
− 0
View file @ bfd7abd0
cmake_minimum_required(VERSION 3.15)
exec_program("git" ${CMAKE_CURRENT_SOURCE_DIR} ARGS "describe --dirty --always --exclude '*'" OUTPUT_VARIABLE SOURCE_VERSION)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
project(biotracker-background_subtraction_tracker
LANGUAGES CXX
......
Src/CMakeLists.txt
+ 3
− 2
View file @ bfd7abd0
......@@ -4,7 +4,7 @@
include(biotracker-utility/TrackerPlugin)
set(target biotracker-backgroundSubtraction.bio_tracker)
set(target python.bio_tracker)
add_behavior_plugin(${target}
"PluginContext.cpp"
......@@ -44,4 +44,5 @@ add_behavior_plugin(${target}
install(TARGETS ${target} OPTIONAL DESTINATION .)
find_package(Qt5 REQUIRED COMPONENTS Xml Network)
target_link_libraries (${target} Qt5::Xml Qt5::Network)
find_package( ZeroMQ REQUIRED )
target_link_libraries (${target} Qt5::Xml Qt5::Network rt zmq)
Src/Model/BioTrackerTrackingAlgorithm.cpp
+ 63
− 161
View file @ bfd7abd0
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <iostream>
#include "BioTrackerTrackingAlgorithm.h"
#include <future>
#include "TrackedComponents/TrackedComponentFactory.h"
#include <chrono>
#include <zmq.hpp>
#include "json.hpp"
using json = nlohmann::json;
void* init_shm_mmap(const char *path, int len) {
int fd = shm_open(path, O_RDWR, 0);
if (fd == -1) {
throw "shm_open";
}
void *shm_buf = mmap(NULL, len,
PROT_READ | PROT_WRITE,
MAP_SHARED, fd, 0);
close(fd);
if (shm_buf == MAP_FAILED) {
throw "mmap";
}
return shm_buf;
}
BioTrackerTrackingAlgorithm::BioTrackerTrackingAlgorithm(IController *parent, IModel* parameter, IModel* trajectory)
: IModelTrackingAlgorithm(parent)
......@@ -13,7 +45,6 @@ BioTrackerTrackingAlgorithm::BioTrackerTrackingAlgorithm(IController *parent, IM
_nn2d = std::make_shared<NN2dMapper>(_TrackedTrajectoryMajor);
_bd = BlobsDetector();
_noFish = -1;
if (_cfg->DoNetwork) {
......@@ -25,8 +56,23 @@ BioTrackerTrackingAlgorithm::BioTrackerTrackingAlgorithm(IController *parent, IM
_lastImage = nullptr;
_lastFramenumber = -1;
_sock = zmq::socket_t(_ctx, zmq::socket_type::req);
_sock.bind("ipc:///tmp/biotracker.python.zmq");
}
void BioTrackerTrackingAlgorithm::request_shared_memory() {
json j = {
{ "type", "request_shared_memory" },
{ "width", _imageX },
{ "height", _imageY },
};
_sock.send(zmq::buffer(j.dump()));
auto res = _sock.recv(_zmq_msg, zmq::recv_flags::none);
auto msg = json::parse(_zmq_msg.to_string_view());
std::string shm_path = msg["path"];
_shm_img = (float*)init_shm_mmap(shm_path.c_str(), _imageX * _imageY * sizeof(float));
}
void BioTrackerTrackingAlgorithm::receiveAreaDescriptorUpdate(IModelAreaDescriptor *areaDescr) {
_AreaInfo = areaDescr;
......@@ -37,114 +83,14 @@ BioTrackerTrackingAlgorithm::~BioTrackerTrackingAlgorithm()
{
}
std::vector<FishPose> BioTrackerTrackingAlgorithm::getLastPositionsAsPose() {
//TODO: This seems kinda fragile: I just assume that the tree has this very certain structure:
// Trajectory -> M Trajectories -> N TrackedElements
// For every of M Trajectories grab the last (highest index) of TrackedElements.
//TODO: If we are tracking somewhere in the middle, this is bad. Do it by id!
std::vector<FishPose> last;
for (int i = 0; i < _TrackedTrajectoryMajor->size(); i++) {
BST::TrackedTrajectory *t = dynamic_cast<BST::TrackedTrajectory *>(_TrackedTrajectoryMajor->getChild(i));
if (t && t->getValid() && !t->getFixed()) {
BST::TrackedElement *e = (BST::TrackedElement *)t->getLastChild();
last.push_back(e->getFishPose());
}
}
return last;
}
void BioTrackerTrackingAlgorithm::refreshPolygon() {
}
void BioTrackerTrackingAlgorithm::receiveParametersChanged() {
if (_lastFramenumber >= 0 && _lastImage && !_lastImage->empty()) {
doTracking(_lastImage, _lastFramenumber);
}
}
void BioTrackerTrackingAlgorithm::sendSelectedImage(std::map<std::string, std::shared_ptr<cv::Mat>> *images) {
std::shared_ptr<cv::Mat> sendImage;
//Send forth whatever the user selected
switch (_TrackingParameter->getSendImage()) {
case 0: //Send none
//sendImage = images.find(std::string("Original"))->second;
//Q_EMIT emitCvMatA(sendImage, QString("Original"));
Q_EMIT emitChangeDisplayImage("Original");
break;
case 1:
sendImage = images->find(std::string("Binarized"))->second;
Q_EMIT emitCvMatA(sendImage, QString("Binarized"));
Q_EMIT emitChangeDisplayImage(QString("Binarized"));
break;
case 2:
sendImage = images->find(std::string("Eroded"))->second;
Q_EMIT emitCvMatA(sendImage, QString("Eroded"));
Q_EMIT emitChangeDisplayImage(QString("Eroded"));
break;
case 3:
sendImage = images->find(std::string("Dilated"))->second;
Q_EMIT emitCvMatA(sendImage, QString("Dilated"));
Q_EMIT emitChangeDisplayImage(QString("Dilated"));
break;
case 4:
sendImage = images->find(std::string("Difference"))->second;
Q_EMIT emitCvMatA(sendImage, QString("Difference"));
Q_EMIT emitChangeDisplayImage(QString("Difference"));
break;
case 5:
sendImage = images->find(std::string("Background"))->second;
Q_EMIT emitCvMatA(sendImage, QString("Background"));
Q_EMIT emitChangeDisplayImage(QString("Background"));
break;
}
}
std::vector<BlobPose> BioTrackerTrackingAlgorithm::getContourCentroids(cv::Mat& image, int minSize){
std::vector<std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
std::vector<BlobPose> centroids;
findContours( image, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE, cv::Point(0, 0) );
for(auto x: contours){
cv::Point2f c(0,0);
float i=0;
for(auto y: x){
c += cv::Point2f(y);
i++;
}
c.x = c.x / i;
c.y = c.y / i;
//cv::RotatedRect minEllipse;
cv::RotatedRect bb = minAreaRect( x );
//check if blob is in tracking area --> this can be optimized by checking earlier (only search blobs in tracking area)
if(!_AreaInfo->inTrackingArea(c)){
continue;
}
BlobPose bc(_AreaInfo->pxToCm(c), c, bb.angle, bb.size.width, bb.size.height);
centroids.push_back(bc);
}
return centroids;
}
#define EEE duration = std::chrono::duration_cast< std::chrono::milliseconds> (std::chrono::steady_clock::now() - startt); block++; std::cout << "Block " << block << ": " << duration.count() << std::endl;
#define SSS startt = std::chrono::steady_clock::now();
void BioTrackerTrackingAlgorithm::doTracking(std::shared_ptr<cv::Mat> p_image, uint framenumber)
{
//Q_EMIT emitCvMatA(p_image, QString("Original"));
//Q_EMIT emitChangeDisplayImage("Original");
//Q_EMIT emitTrackingDone(framenumber);
//return;
int block = 0;
_ipp.m_TrackingParameter = _TrackingParameter;
_lastImage = p_image;
_lastFramenumber = framenumber;
......@@ -157,77 +103,33 @@ void BioTrackerTrackingAlgorithm::doTracking(std::shared_ptr<cv::Mat> p_image, u
if (_imageX != p_image->size().width || _imageY != p_image->size().height) {
_imageX = p_image->size().width;
_imageY = p_image->size().height;
request_shared_memory();
Q_EMIT emitDimensionUpdate(_imageX, _imageY);
}
std::chrono::system_clock::time_point start = std::chrono::system_clock::now();
//Refuse to run tracking if we have no area info...
if (_AreaInfo == nullptr) {
Q_EMIT emitTrackingDone(framenumber);
return;
}
//The user changed the # of fish. Reset the history and start over!
if (_noFish != _TrackedTrajectoryMajor->validCount()) {
_noFish = _TrackedTrajectoryMajor->validCount();
//resetFishHistory(_noFish);
_nn2d = std::make_shared<NN2dMapper>(_TrackedTrajectoryMajor);
}
if (_TrackingParameter->getResetBackground()) {
_TrackingParameter->setResetBackground(false);
_ipp.resetBackgroundImage();
}
//Do the preprocessing
std::map<std::string, std::shared_ptr<cv::Mat>> images = _ipp.preProcess(p_image);
std::shared_ptr<cv::Mat> dilated = images.find(std::string("Dilated"))->second;
std::shared_ptr<cv::Mat> greyMat = images.find(std::string("Greyscale"))->second;
//Find blobs via ellipsefitting
_bd.setMaxBlobSize(_TrackingParameter->getMaxBlobSize());
_bd.setMinBlobSize(_TrackingParameter->getMinBlobSize());
//std::vector<BlobPose> blobs = _bd.getPoses(*dilated, *greyMat);
std::vector<BlobPose> blobs = getContourCentroids(*dilated, 111);
// Never switch the position of the trajectories. The NN2d mapper relies on this!
// If you mess up the order, add or remove some t, then create a new mapper.
std::vector<FishPose> fish = getLastPositionsAsPose();
//Find new positions using 2D nearest neighbor
std::tuple<std::vector<FishPose>, std::vector<float>> poses = _nn2d->getNewPoses(_TrackedTrajectoryMajor, framenumber, blobs);
//Insert new poses into data structure
int trajNumber = 0;
for (int i = 0; i < _TrackedTrajectoryMajor->size(); i++) {
BST::TrackedTrajectory *t = dynamic_cast<BST::TrackedTrajectory *>(_TrackedTrajectoryMajor->getChild(i));
if (t && t->getValid() && !t->getFixed()) {
BST::TrackedElement *e = new BST::TrackedElement(t, "n.a.", t->getId());
e->setFishPose(std::get<0>(poses)[trajNumber]);
e->setTime(start);
t->add(e, framenumber);
trajNumber++;
}
}
//Send forth new positions to the robotracker, if networking is enabled
if (_TrackingParameter->getDoNetwork()){
std::vector<FishPose> ps = std::get<0>(poses);
_listener->sendPositions(framenumber, ps, std::vector<cv::Point2f>(), start);
}
sendSelectedImage(&images);
//First the user still wants to see the original image, right?
if (framenumber==1) {
Q_EMIT emitChangeDisplayImage("Original");
}
cv::Mat grayscale, float_img;
cv::cvtColor(*p_image, grayscale, cv::COLOR_BGR2GRAY);
grayscale.convertTo(float_img, CV_32F);
float *img_data = float_img.ptr<float>(0);
int img_data_len = float_img.cols * float_img.rows * sizeof(float);
memcpy(_shm_img, img_data, img_data_len);
json j = {
{ "type", "predict_frame" },
{ "frame_id", framenumber },
};
_sock.send(zmq::buffer(j.dump()));
auto res = _sock.recv(_zmq_msg, zmq::recv_flags::none);
auto msg = json::parse(_zmq_msg.to_string_view());
std::string newSel = _TrackingParameter->getNewSelection();
Q_EMIT emitChangeDisplayImage("Original");
Q_EMIT emitTrackingDone(framenumber);
}
Src/Model/BioTrackerTrackingAlgorithm.h
+ 7
− 5
View file @ bfd7abd0
......@@ -17,6 +17,7 @@
#include "TrackingAlgorithm/NN2dMapper.h"
#include "Interfaces/IModel/IModelAreaDescriptor.h"
#include <iostream>
#include <zmq.hpp>
#include "Network/TcpListener.h"
#include "../Config.h"
......@@ -40,11 +41,7 @@ public Q_SLOTS:
void receiveParametersChanged();
private:
std::vector<BlobPose> getContourCentroids(cv::Mat& image, int minSize);
void refreshPolygon();
void sendSelectedImage(std::map<std::string, std::shared_ptr<cv::Mat>>* images);
std::vector<FishPose> getLastPositionsAsPose();
void request_shared_memory();
BST::TrackedTrajectory* _TrackedTrajectoryMajor;
TrackerParameter* _TrackingParameter;
......@@ -69,6 +66,11 @@ std::vector<BlobPose> getContourCentroids(cv::Mat& image, int minSize);
std::shared_ptr<cv::Mat> _lastImage;
uint _lastFramenumber;
Config *_cfg;
zmq::context_t _ctx;
zmq::socket_t _sock;
zmq::message_t _zmq_msg;
float *_shm_img;
};
#endif // BIOTRACKERTRACKINGALGORITHM_H
Src/Model/json.hpp 0 → 100644
+ 26137
− 0
View file @ bfd7abd0
Source diff could not be displayed: it is too large. Options to address this: view the blob.
cmake/FindZeroMQ.cmake 0 → 100644
+ 26
− 0
View file @ bfd7abd0
set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
find_package(PkgConfig)
pkg_check_modules(PC_LIBZMQ QUIET libzmq)
set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION})
find_library(ZeroMQ_LIBRARY NAMES libzmq.so libzmq.dylib libzmq.dll
PATHS ${PC_LIBZMQ_LIBDIR} ${PC_LIBZMQ_LIBRARY_DIRS})
find_library(ZeroMQ_STATIC_LIBRARY NAMES libzmq-static.a libzmq.a libzmq.dll.a
PATHS ${PC_LIBZMQ_LIBDIR} ${PC_LIBZMQ_LIBRARY_DIRS})
if(ZeroMQ_LIBRARY OR ZeroMQ_STATIC_LIBRARY)
set(ZeroMQ_FOUND ON)
endif()
if (TARGET libzmq)
# avoid errors defining targets twice
return()
endif()
add_library(libzmq SHARED IMPORTED)
set_property(TARGET libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq PROPERTY IMPORTED_LOCATION ${ZeroMQ_LIBRARY})
add_library(libzmq-static STATIC IMPORTED ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq-static PROPERTY IMPORTED_LOCATION ${ZeroMQ_STATIC_LIBRARY})
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment