OpenShot Library | libopenshot  0.3.0
Tracker.cpp
Go to the documentation of this file.
1 
10 // Copyright (c) 2008-2019 OpenShot Studios, LLC
11 //
12 // SPDX-License-Identifier: LGPL-3.0-or-later
13 
14 #include <string>
15 #include <memory>
16 #include <fstream>
17 #include <iostream>
18 
19 #include "effects/Tracker.h"
20 #include "Exceptions.h"
21 #include "Timeline.h"
22 #include "trackerdata.pb.h"
23 
24 #include <google/protobuf/util/time_util.h>
25 
26 #include <QImage>
27 #include <QPainter>
28 #include <QRectF>
29 
30 using namespace std;
31 using namespace openshot;
32 using google::protobuf::util::TimeUtil;
33 
35 Tracker::Tracker(std::string clipTrackerDataPath)
36 {
37  // Init effect properties
38  init_effect_details();
39  // Instantiate a TrackedObjectBBox object and point to it
40  TrackedObjectBBox trackedDataObject;
41  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
42  // Tries to load the tracked object's data from protobuf file
43  trackedData->LoadBoxData(clipTrackerDataPath);
44  ClipBase* parentClip = this->ParentClip();
45  trackedData->ParentClip(parentClip);
46  trackedData->Id(std::to_string(0));
47  // Insert TrackedObject with index 0 to the trackedObjects map
48  trackedObjects.insert({0, trackedData});
49 }
50 
51 // Default constructor
52 Tracker::Tracker()
53 {
54  // Init effect properties
55  init_effect_details();
56  // Instantiate a TrackedObjectBBox object and point to it
57  TrackedObjectBBox trackedDataObject;
58  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59  ClipBase* parentClip = this->ParentClip();
60  trackedData->ParentClip(parentClip);
61  trackedData->Id(std::to_string(0));
62  // Insert TrackedObject with index 0 to the trackedObjects map
63  trackedObjects.insert({0, trackedData});
64 }
65 
66 
67 // Init effect settings
68 void Tracker::init_effect_details()
69 {
71  InitEffectInfo();
72 
74  info.class_name = "Tracker";
75  info.name = "Tracker";
76  info.description = "Track the selected bounding box through the video.";
77  info.has_audio = false;
78  info.has_video = true;
79  info.has_tracked_object = true;
80 
81  this->TimeScale = 1.0;
82 }
83 
84 // This method is required for all derived classes of EffectBase, and returns a
85 // modified openshot::Frame object
86 std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
87 {
88  // Get the frame's image
89  cv::Mat frame_image = frame->GetImageCV();
90 
91  // Initialize the Qt rectangle that will hold the positions of the bounding-box
92  QRectF boxRect;
93  // Initialize the image of the TrackedObject child clip
94  std::shared_ptr<QImage> childClipImage = nullptr;
95 
96  // Check if frame isn't NULL
97  if(!frame_image.empty() &&
98  trackedData->Contains(frame_number) &&
99  trackedData->visible.GetValue(frame_number) == 1)
100  {
101  // Get the width and height of the image
102  float fw = frame_image.size().width;
103  float fh = frame_image.size().height;
104 
105  // Get the bounding-box of given frame
106  BBox fd = trackedData->GetBox(frame_number);
107 
108  // Check if track data exists for the requested frame
109  if (trackedData->draw_box.GetValue(frame_number) == 1)
110  {
111  std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
112  int stroke_width = trackedData->stroke_width.GetValue(frame_number);
113  float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
114  std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
115  float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
116 
117  // Create a rotated rectangle object that holds the bounding box
118  cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
119  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
120  (int) (fd.angle) );
121 
122  DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
123  DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
124  }
125 
126  // Get the image of the Tracked Object' child clip
127  if (trackedData->ChildClipId() != ""){
128  // Cast the parent timeline of this effect
129  Timeline* parentTimeline = (Timeline *) ParentTimeline();
130  if (parentTimeline){
131  // Get the Tracked Object's child clip
132  Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
133  if (childClip){
134  // Get the image of the child clip for this frame
135  std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
136  std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
137  childClipImage = childClipFrame->GetImage();
138 
139  // Set the Qt rectangle with the bounding-box properties
140  boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
141  (int)((fd.cy - fd.height/2)*fh),
142  (int)(fd.width*fw),
143  (int)(fd.height*fh) );
144  }
145  }
146  }
147 
148  }
149 
150  // Set image with drawn box to frame
151  // If the input image is NULL or doesn't have tracking data, it's returned as it came
152  frame->SetImageCV(frame_image);
153 
154  // Set the bounding-box image with the Tracked Object's child clip image
155  if (childClipImage){
156  // Get the frame image
157  QImage frameImage = *(frame->GetImage());
158 
159  // Set a Qt painter to the frame image
160  QPainter painter(&frameImage);
161 
162  // Draw the child clip image inside the bounding-box
163  painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
164 
165  // Set the frame image as the composed image
166  frame->AddImage(std::make_shared<QImage>(frameImage));
167  }
168 
169  return frame;
170 }
171 
172 void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
173  // Get the bouding box vertices
174  cv::Point2f vertices2f[4];
175  box.points(vertices2f);
176 
177  // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
178  // select min enclosing rectangle to draw on a small portion of the image
179  // cv::Rect rect = box.boundingRect();
180  // cv::Mat image = frame_image(rect)
181 
182  if(is_background){
183  cv::Mat overlayFrame;
184  frame_image.copyTo(overlayFrame);
185 
186  // draw bounding box background
187  cv::Point vertices[4];
188  for(int i = 0; i < 4; ++i){
189  vertices[i] = vertices2f[i];}
190 
191  cv::Rect rect = box.boundingRect();
192  cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
193  // add opacity
194  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
195  }
196  else{
197  cv::Mat overlayFrame;
198  frame_image.copyTo(overlayFrame);
199 
200  // Draw bounding box
201  for (int i = 0; i < 4; i++)
202  {
203  cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
204  thickness, cv::LINE_AA);
205  }
206 
207  // add opacity
208  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
209  }
210 }
211 
212 // Get the indexes and IDs of all visible objects in the given frame
213 std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
214 
215  // Initialize the JSON objects
216  Json::Value root;
217  root["visible_objects_index"] = Json::Value(Json::arrayValue);
218  root["visible_objects_id"] = Json::Value(Json::arrayValue);
219 
220  // Iterate through the tracked objects
221  for (const auto& trackedObject : trackedObjects){
222  // Get the tracked object JSON properties for this frame
223  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
224  if (trackedObjectJSON["visible"]["value"].asBool()){
225  // Save the object's index and ID if it's visible in this frame
226  root["visible_objects_index"].append(trackedObject.first);
227  root["visible_objects_id"].append(trackedObject.second->Id());
228  }
229  }
230 
231  return root.toStyledString();
232 }
233 
234 // Generate JSON string of this object
235 std::string Tracker::Json() const {
236 
237  // Return formatted string
238  return JsonValue().toStyledString();
239 }
240 
241 // Generate Json::Value for this object
242 Json::Value Tracker::JsonValue() const {
243 
244  // Create root json object
245  Json::Value root = EffectBase::JsonValue(); // get parent properties
246 
247  // Save the effect's properties on root
248  root["type"] = info.class_name;
249  root["protobuf_data_path"] = protobuf_data_path;
250  root["BaseFPS"]["num"] = BaseFPS.num;
251  root["BaseFPS"]["den"] = BaseFPS.den;
252  root["TimeScale"] = this->TimeScale;
253 
254  // Add trackedObjects IDs to JSON
255  Json::Value objects;
256  for (auto const& trackedObject : trackedObjects){
257  Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
258  // add object json
259  objects[trackedObject.second->Id()] = trackedObjectJSON;
260  }
261  root["objects"] = objects;
262 
263  // return JsonValue
264  return root;
265 }
266 
267 // Load JSON string into this object
268 void Tracker::SetJson(const std::string value) {
269 
270  // Parse JSON string into JSON objects
271  try
272  {
273  const Json::Value root = openshot::stringToJson(value);
274  // Set all values that match
275  SetJsonValue(root);
276  }
277  catch (const std::exception& e)
278  {
279  // Error parsing JSON (or missing keys)
280  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
281  }
282  return;
283 }
284 
285 // Load Json::Value into this object
286 void Tracker::SetJsonValue(const Json::Value root) {
287 
288  // Set parent data
289  EffectBase::SetJsonValue(root);
290 
291  if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
292  {
293  if (!root["BaseFPS"]["num"].isNull())
294  {
295  BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
296  }
297  if (!root["BaseFPS"]["den"].isNull())
298  {
299  BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
300  }
301  }
302 
303  if (!root["TimeScale"].isNull())
304  TimeScale = (double) root["TimeScale"].asDouble();
305 
306  // Set data from Json (if key is found)
307  if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
308  {
309  protobuf_data_path = root["protobuf_data_path"].asString();
310  if(!trackedData->LoadBoxData(protobuf_data_path))
311  {
312  std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
313  protobuf_data_path = "";
314  }
315  }
316 
317  if (!root["objects"].isNull()){
318  for (auto const& trackedObject : trackedObjects){
319  std::string obj_id = std::to_string(trackedObject.first);
320  if(!root["objects"][obj_id].isNull()){
321  trackedObject.second->SetJsonValue(root["objects"][obj_id]);
322  }
323  }
324  }
325 
326  // Set the tracked object's ids
327  if (!root["objects_id"].isNull()){
328  for (auto const& trackedObject : trackedObjects){
329  Json::Value trackedObjectJSON;
330  trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
331  trackedObject.second->SetJsonValue(trackedObjectJSON);
332  }
333  }
334 
335  return;
336 }
337 
338 // Get all properties for a specific frame
339 std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
340 
341  // Generate JSON properties list
342  Json::Value root;
343 
344  // Add trackedObject properties to JSON
345  Json::Value objects;
346  for (auto const& trackedObject : trackedObjects){
347  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
348  // add object json
349  objects[trackedObject.second->Id()] = trackedObjectJSON;
350  }
351  root["objects"] = objects;
352 
353  // Append effect's properties
354  root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
355  root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
356  root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
357  root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
358  root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
359  root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
360 
361  // Return formatted string
362  return root.toStyledString();
363 }
Header file for Tracker effect class.
float cy
y-coordinate of the bounding box center
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:90
float height
bounding box height
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:408
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
float angle
bounding box rotation angle [degrees]
Header file for Timeline class.
Header file for all Exception classes.
bool LoadBoxData(std::string inputFilePath)
Load the bounding-boxes information from the protobuf file.
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:90
float width
bounding box width
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:33
This struct holds the information of a bounding-box.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
float cx
x-coordinate of the bounding box center
Exception for invalid JSON.
Definition: Exceptions.h:217
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:378
This class contains the properties of a tracked object and functions to manipulate it...
This class represents a timeline.
Definition: Timeline.h:150