21 #include "objdetectdata.pb.h" 31 ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
34 init_effect_details();
37 LoadObjDetectdData(clipObDetectDataPath);
40 selectedObjectIndex = trackedObjects.begin()->first;
44 ObjectDetection::ObjectDetection()
47 init_effect_details();
50 selectedObjectIndex = trackedObjects.begin()->first;
54 void ObjectDetection::init_effect_details()
60 info.class_name =
"ObjectDetection";
61 info.name =
"Object Detector";
62 info.description =
"Detect objects through the video.";
63 info.has_audio =
false;
64 info.has_video =
true;
65 info.has_tracked_object =
true;
70 std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
73 cv::Mat cv_image = frame->GetImageCV();
81 std::vector<QRectF> boxRects;
83 std::vector<std::shared_ptr<QImage>> childClipImages;
86 if (detectionsData.find(frame_number) != detectionsData.end()) {
87 float fw = cv_image.size().width;
88 float fh = cv_image.size().height;
91 for(
int i = 0; i<detections.
boxes.size(); i++){
94 if(detections.
confidences.at(i) < confidence_threshold){
98 if( display_classes.size() > 0 &&
99 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
104 int objectId = detections.
objectIds.at(i);
107 auto trackedObject_it = trackedObjects.find(objectId);
110 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<
TrackedObjectBBox>(trackedObject_it->second);
113 if (trackedObject->Contains(frame_number) &&
114 trackedObject->visible.GetValue(frame_number) == 1)
117 BBox trackedBox = trackedObject->GetBox(frame_number);
118 bool draw_text = !display_box_text.GetValue(frame_number);
119 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
120 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
121 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
122 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
123 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
135 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
136 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
137 (
int)( trackedBox.
width*fw),
138 (
int)( trackedBox.
height*fh)
142 if (trackedObject->draw_box.GetValue(frame_number) == 0)
149 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
151 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
155 if (trackedObject->ChildClipId() !=
""){
160 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
163 std::shared_ptr<Frame> f(
new Frame(1, frame->GetWidth(), frame->GetHeight(),
"#00000000"));
165 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(f, frame_number);
166 childClipImages.push_back(childClipFrame->GetImage());
170 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
171 (
int)((trackedBox.
cy - trackedBox.
height/2)*fh),
172 (
int)(trackedBox.
width*fw),
173 (
int)(trackedBox.
height*fh));
174 boxRects.push_back(boxRect);
183 frame->SetImageCV(cv_image);
186 if(boxRects.size() > 0){
188 QImage frameImage = *(frame->GetImage());
189 for(
int i; i < boxRects.size();i++){
191 QPainter painter(&frameImage);
193 painter.drawImage(boxRects[i], *childClipImages[i], QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
196 frame->AddImage(std::make_shared<QImage>(frameImage));
202 void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
203 int thickness,
bool is_background){
205 cv::Point2f vertices2f[4];
206 box.points(vertices2f);
214 cv::Mat overlayFrame;
215 frame_image.copyTo(overlayFrame);
218 cv::Point vertices[4];
219 for(
int i = 0; i < 4; ++i){
220 vertices[i] = vertices2f[i];}
222 cv::Rect rect = box.boundingRect();
223 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
225 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
228 cv::Mat overlayFrame;
229 frame_image.copyTo(overlayFrame);
232 for (
int i = 0; i < 4; i++)
234 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
235 thickness, cv::LINE_AA);
239 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
243 void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
244 float alpha,
int thickness,
bool is_background,
bool display_text)
248 cv::Mat overlayFrame;
249 frame.copyTo(overlayFrame);
252 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
255 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
258 cv::Mat overlayFrame;
259 frame.copyTo(overlayFrame);
262 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
266 std::string label = cv::format(
"%.2f", conf);
267 if (!classNames.empty())
269 CV_Assert(classId < (
int)classNames.size());
270 label = classNames[classId] +
":" + label;
275 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
278 double top = std::max((
int)box.y, labelSize.height);
280 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
281 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
282 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
285 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
290 bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
292 pb_objdetect::ObjDetect objMessage;
295 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
296 if (!objMessage.ParseFromIstream(&input)) {
297 std::cerr <<
"Failed to parse protobuf message." << std::endl;
303 detectionsData.clear();
304 trackedObjects.clear();
309 for(
int i = 0; i < objMessage.classnames_size(); i++)
311 classNames.push_back(objMessage.classnames(i));
312 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
316 for (
size_t i = 0; i < objMessage.frame_size(); i++)
319 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
322 size_t id = pbFrameData.id();
325 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
328 std::vector<int> classIds;
329 std::vector<float> confidences;
330 std::vector<cv::Rect_<float>> boxes;
331 std::vector<int> objectIds;
334 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
337 float x = pBox.Get(i).x();
338 float y = pBox.Get(i).y();
339 float w = pBox.Get(i).w();
340 float h = pBox.Get(i).h();
342 int classId = pBox.Get(i).classid();
344 float confidence = pBox.Get(i).confidence();
347 int objectId = pBox.Get(i).objectid();
350 auto trackedObject = trackedObjects.find(objectId);
352 if (trackedObject != trackedObjects.end())
355 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
360 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
361 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
363 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
364 ClipBase* parentClip = this->ParentClip();
365 trackedObjPtr->ParentClip(parentClip);
369 trackedObjPtr->Id(std::to_string(objectId));
370 trackedObjects.insert({objectId, trackedObjPtr});
374 cv::Rect_<float> box(x, y, w, h);
377 boxes.push_back(box);
378 classIds.push_back(classId);
379 confidences.push_back(confidence);
380 objectIds.push_back(objectId);
384 detectionsData[id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
388 google::protobuf::ShutdownProtobufLibrary();
394 std::string ObjectDetection::GetVisibleObjects(int64_t frame_number)
const{
398 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
399 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
402 if (detectionsData.find(frame_number) == detectionsData.end()){
403 return root.toStyledString();
408 for(
int i = 0; i<detections.
boxes.size(); i++){
410 if(detections.
confidences.at(i) < confidence_threshold){
415 if( display_classes.size() > 0 &&
416 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
420 int objectId = detections.
objectIds.at(i);
422 auto trackedObject = trackedObjects.find(objectId);
425 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
427 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
428 trackedObject->second->ExactlyContains(frame_number)){
430 root[
"visible_objects_index"].append(trackedObject->first);
431 root[
"visible_objects_id"].append(trackedObject->second->Id());
435 return root.toStyledString();
439 std::string ObjectDetection::Json()
const {
442 return JsonValue().toStyledString();
446 Json::Value ObjectDetection::JsonValue()
const {
449 Json::Value root = EffectBase::JsonValue();
450 root[
"type"] = info.class_name;
451 root[
"protobuf_data_path"] = protobuf_data_path;
452 root[
"selected_object_index"] = selectedObjectIndex;
453 root[
"confidence_threshold"] = confidence_threshold;
454 root[
"display_box_text"] = display_box_text.JsonValue();
458 for (
auto const& trackedObject : trackedObjects){
459 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
461 objects[trackedObject.second->Id()] = trackedObjectJSON;
463 root[
"objects"] = objects;
470 void ObjectDetection::SetJson(
const std::string value) {
479 catch (
const std::exception& e)
482 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
487 void ObjectDetection::SetJsonValue(
const Json::Value root) {
489 EffectBase::SetJsonValue(root);
492 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
493 protobuf_data_path = root[
"protobuf_data_path"].asString();
495 if(!LoadObjDetectdData(protobuf_data_path)){
496 throw InvalidFile(
"Invalid protobuf data path",
"");
497 protobuf_data_path =
"";
502 if (!root[
"selected_object_index"].isNull())
503 selectedObjectIndex = root[
"selected_object_index"].asInt();
505 if (!root[
"confidence_threshold"].isNull())
506 confidence_threshold = root[
"confidence_threshold"].asFloat();
508 if (!root[
"display_box_text"].isNull())
509 display_box_text.SetJsonValue(root[
"display_box_text"]);
511 if (!root[
"class_filter"].isNull()){
512 class_filter = root[
"class_filter"].asString();
513 std::stringstream ss(class_filter);
514 display_classes.clear();
519 std::getline( ss, substr,
',' );
520 display_classes.push_back( substr );
524 if (!root[
"objects"].isNull()){
525 for (
auto const& trackedObject : trackedObjects){
526 std::string obj_id = std::to_string(trackedObject.first);
527 if(!root[
"objects"][obj_id].isNull()){
528 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
534 if (!root[
"objects_id"].isNull()){
535 for (
auto const& trackedObject : trackedObjects){
536 Json::Value trackedObjectJSON;
537 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
538 trackedObject.second->SetJsonValue(trackedObjectJSON);
544 std::string ObjectDetection::PropertiesJSON(int64_t requested_frame)
const {
550 if(trackedObjects.count(selectedObjectIndex) != 0){
551 auto selectedObject = trackedObjects.at(selectedObjectIndex);
553 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
555 objects[selectedObject->Id()] = trackedObjectJSON;
558 root[
"objects"] = objects;
560 root[
"selected_object_index"] = add_property_json(
"Selected Object", selectedObjectIndex,
"int",
"", NULL, 0, 200,
false, requested_frame);
561 root[
"id"] = add_property_json(
"ID", 0.0,
"string", Id(), NULL, -1, -1,
true, requested_frame);
562 root[
"position"] = add_property_json(
"Position", Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
563 root[
"layer"] = add_property_json(
"Track", Layer(),
"int",
"", NULL, 0, 20,
false, requested_frame);
564 root[
"start"] = add_property_json(
"Start", Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
565 root[
"end"] = add_property_json(
"End", End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
566 root[
"duration"] = add_property_json(
"Duration", Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
567 root[
"confidence_threshold"] = add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
568 root[
"class_filter"] = add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
570 root[
"display_box_text"] = add_property_json(
"Draw Box Text", display_box_text.GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
571 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"Off", 1, display_box_text.GetValue(requested_frame)));
572 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"On", 0, display_box_text.GetValue(requested_frame)));
575 return root.toStyledString();
std::vector< cv::Rect_< float > > boxes
Header file for Tracker effect class.
float cy
y-coordinate of the bounding box center
This class represents a single frame of video (i.e. image & audio data)
float height
bounding box height
std::vector< int > classIds
Header file for Object Detection effect class.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
const Json::Value stringToJson(const std::string value)
Header file for Timeline class.
std::vector< int > objectIds
Header file for all Exception classes.
This class represents a clip (used to arrange readers on the timeline)
Exception for files that can not be found or opened.
float width
bounding box width
This abstract class is the base class, used by all clips in libopenshot.
This struct holds the information of a bounding-box.
This namespace is the default namespace for all code in the openshot library.
float cx
x-coordinate of the bounding box center
std::vector< float > confidences
Exception for invalid JSON.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
This class contains the properties of a tracked object and functions to manipulate it...
This class represents a timeline.