OpenShot Library | libopenshot 0.3.3
Loading...
Searching...
No Matches
CVTracker.cpp
Go to the documentation of this file.
1
10// Copyright (c) 2008-2019 OpenShot Studios, LLC
11//
12// SPDX-License-Identifier: LGPL-3.0-or-later
13
14#include <fstream>
15#include <iomanip>
16#include <iostream>
17
18#include <google/protobuf/util/time_util.h>
19
20#include "OpenCVUtilities.h"
21#include "CVTracker.h"
22#include "trackerdata.pb.h"
23#include "Exceptions.h"
24
25using namespace openshot;
26using google::protobuf::util::TimeUtil;
27
28// Constructor
29CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController)
30: processingController(&processingController), json_interval(false){
31 SetJson(processInfoJson);
32 start = 1;
33 end = 1;
34}
35
36// Set desirable tracker method
37cv::Ptr<OPENCV_TRACKER_TYPE> CVTracker::selectTracker(std::string trackerType){
38
39 if (trackerType == "BOOSTING")
40 return OPENCV_TRACKER_NS::TrackerBoosting::create();
41 if (trackerType == "MIL")
42 return OPENCV_TRACKER_NS::TrackerMIL::create();
43 if (trackerType == "KCF")
44 return OPENCV_TRACKER_NS::TrackerKCF::create();
45 if (trackerType == "TLD")
46 return OPENCV_TRACKER_NS::TrackerTLD::create();
47 if (trackerType == "MEDIANFLOW")
48 return OPENCV_TRACKER_NS::TrackerMedianFlow::create();
49 if (trackerType == "MOSSE")
50 return OPENCV_TRACKER_NS::TrackerMOSSE::create();
51 if (trackerType == "CSRT")
52 return OPENCV_TRACKER_NS::TrackerCSRT::create();
53
54 return nullptr;
55}
56
57// Track object in the hole clip or in a given interval
58void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
59
60 video.Open();
61 if(!json_interval){
62 start = _start; end = _end;
63
64 if(!process_interval || end <= 1 || end-start == 0){
65 // Get total number of frames in video
66 start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
67 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
68 }
69 }
70 else{
71 start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
72 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
73 }
74
75 if(error){
76 return;
77 }
78
79 processingController->SetError(false, "");
80 bool trackerInit = false;
81
82 size_t frame;
83 // Loop through video
84 for (frame = start; frame <= end; frame++)
85 {
86
87 // Stop the feature tracker process
88 if(processingController->ShouldStop()){
89 return;
90 }
91
92 size_t frame_number = frame;
93 // Get current frame
94 std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
95
96 // Grab OpenCV Mat image
97 cv::Mat cvimage = f->GetImageCV();
98
99 if(frame == start){
100 // Take the normalized inital bounding box and multiply to the current video shape
101 bbox = cv::Rect2d(int(bbox.x*cvimage.cols), int(bbox.y*cvimage.rows),
102 int(bbox.width*cvimage.cols), int(bbox.height*cvimage.rows));
103 }
104
105 // Pass the first frame to initialize the tracker
106 if(!trackerInit){
107
108 // Initialize the tracker
109 initTracker(cvimage, frame_number);
110
111 trackerInit = true;
112 }
113 else{
114 // Update the object tracker according to frame
115 trackerInit = trackFrame(cvimage, frame_number);
116
117 // Draw box on image
118 FrameData fd = GetTrackedData(frame_number);
119
120 }
121 // Update progress
122 processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
123 }
124}
125
126// Initialize the tracker
127bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
128
129 // Create new tracker object
130 tracker = selectTracker(trackerType);
131
132 // Correct if bounding box contains negative proportions (width and/or height < 0)
133 if(bbox.width < 0){
134 bbox.x = bbox.x - abs(bbox.width);
135 bbox.width = abs(bbox.width);
136 }
137 if(bbox.height < 0){
138 bbox.y = bbox.y - abs(bbox.height);
139 bbox.height = abs(bbox.height);
140 }
141
142 // Initialize tracker
143 tracker->init(frame, bbox);
144
145 float fw = frame.size().width;
146 float fh = frame.size().height;
147
148 // Add new frame data
149 trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw,
150 (bbox.y)/fh,
151 (bbox.x+bbox.width)/fw,
152 (bbox.y+bbox.height)/fh);
153
154 return true;
155}
156
157// Update the object tracker according to frame
158bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
159 // Update the tracking result
160 bool ok = tracker->update(frame, bbox);
161
162 // Add frame number and box coords if tracker finds the object
163 // Otherwise add only frame number
164 if (ok)
165 {
166 float fw = frame.size().width;
167 float fh = frame.size().height;
168
169 cv::Rect2d filtered_box = filter_box_jitter(frameId);
170 // Add new frame data
171 trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw,
172 (filtered_box.y)/fh,
173 (filtered_box.x+filtered_box.width)/fw,
174 (filtered_box.y+filtered_box.height)/fh);
175 }
176 else
177 {
178 // Copy the last frame data if the tracker get lost
179 trackedDataById[frameId] = trackedDataById[frameId-1];
180 }
181
182 return ok;
183}
184
185cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){
186 // get tracked data for the previous frame
187 float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1;
188 float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1;
189
190 float curr_box_width = bbox.width;
191 float curr_box_height = bbox.height;
192 // keep the last width and height if the difference is less than 1%
193 float threshold = 0.01;
194
195 cv::Rect2d filtered_box = bbox;
196 if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){
197 filtered_box.width = last_box_width;
198 }
199 if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){
200 filtered_box.height = last_box_height;
201 }
202 return filtered_box;
203}
204
206 using std::ios;
207
208 // Create tracker message
209 pb_tracker::Tracker trackerMessage;
210
211 // Iterate over all frames data and save in protobuf message
212 for(std::map<size_t,FrameData>::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){
213 FrameData fData = it->second;
214 pb_tracker::Frame* pbFrameData;
215 AddFrameDataToProto(trackerMessage.add_frame(), fData);
216 }
217
218 // Add timestamp
219 *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
220
221 {
222 // Write the new message to disk.
223 std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
224 if (!trackerMessage.SerializeToOstream(&output)) {
225 std::cerr << "Failed to write protobuf message." << std::endl;
226 return false;
227 }
228 }
229
230 // Delete all global objects allocated by libprotobuf.
231 google::protobuf::ShutdownProtobufLibrary();
232
233 return true;
234
235}
236
237// Add frame tracked data into protobuf message.
238void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) {
239
240 // Save frame number and rotation
241 pbFrameData->set_id(fData.frame_id);
242 pbFrameData->set_rotation(0);
243
244 pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
245 // Save bounding box data
246 box->set_x1(fData.x1);
247 box->set_y1(fData.y1);
248 box->set_x2(fData.x2);
249 box->set_y2(fData.y2);
250}
251
252// Get tracker info for the desired frame
254
255 // Check if the tracker info for the requested frame exists
256 if ( trackedDataById.find(frameId) == trackedDataById.end() ) {
257
258 return FrameData();
259 } else {
260
261 return trackedDataById[frameId];
262 }
263
264}
265
266// Load JSON string into this object
267void CVTracker::SetJson(const std::string value) {
268 // Parse JSON string into JSON objects
269 try
270 {
271 const Json::Value root = openshot::stringToJson(value);
272 // Set all values that match
273
274 SetJsonValue(root);
275 }
276 catch (const std::exception& e)
277 {
278 // Error parsing JSON (or missing keys)
279 throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
280 }
281}
282
283// Load Json::Value into this object
284void CVTracker::SetJsonValue(const Json::Value root) {
285
286 // Set data from Json (if key is found)
287 if (!root["protobuf_data_path"].isNull()){
288 protobuf_data_path = (root["protobuf_data_path"].asString());
289 }
290 if (!root["tracker-type"].isNull()){
291 trackerType = (root["tracker-type"].asString());
292 }
293
294 if (!root["region"].isNull()){
295 double x = root["region"]["normalized_x"].asDouble();
296 double y = root["region"]["normalized_y"].asDouble();
297 double w = root["region"]["normalized_width"].asDouble();
298 double h = root["region"]["normalized_height"].asDouble();
299 cv::Rect2d prev_bbox(x,y,w,h);
300 bbox = prev_bbox;
301
302 if (!root["region"]["first-frame"].isNull()){
303 start = root["region"]["first-frame"].asInt64();
304 json_interval = true;
305 }
306 else{
307 processingController->SetError(true, "No first-frame");
308 error = true;
309 }
310
311 }
312 else{
313 processingController->SetError(true, "No initial bounding box selected");
314 error = true;
315 }
316
317}
318
319/*
320||||||||||||||||||||||||||||||||||||||||||||||||||
321 ONLY FOR MAKE TEST
322||||||||||||||||||||||||||||||||||||||||||||||||||
323*/
324
325// Load protobuf data file
327 using std::ios;
328
329 // Create tracker message
330 pb_tracker::Tracker trackerMessage;
331
332 {
333 // Read the existing tracker message.
334 std::fstream input(protobuf_data_path, ios::in | ios::binary);
335 if (!trackerMessage.ParseFromIstream(&input)) {
336 std::cerr << "Failed to parse protobuf message." << std::endl;
337 return false;
338 }
339 }
340
341 // Make sure the trackedData is empty
342 trackedDataById.clear();
343
344 // Iterate over all frames of the saved message
345 for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
346 const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i);
347
348 // Load frame and rotation data
349 size_t id = pbFrameData.id();
350 float rotation = pbFrameData.rotation();
351
352 // Load bounding box data
353 const pb_tracker::Frame::Box& box = pbFrameData.bounding_box();
354 float x1 = box.x1();
355 float y1 = box.y1();
356 float x2 = box.x2();
357 float y2 = box.y2();
358
359 // Assign data to tracker map
360 trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
361 }
362
363 // Delete all global objects allocated by libprotobuf.
364 google::protobuf::ShutdownProtobufLibrary();
365
366 return true;
367}
Track an object selected by the user.
Header file for all Exception classes.
Header file for OpenCVUtilities (set some common macros)
void SetError(bool err, std::string message)
void trackClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Definition CVTracker.cpp:58
CVTracker(std::string processInfoJson, ProcessingController &processingController)
Definition CVTracker.cpp:29
bool SaveTrackedData()
Save protobuf file.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
void SetJson(const std::string value)
Load JSON string into this object.
cv::Rect2d filter_box_jitter(size_t frameId)
Filter current bounding box jitter.
void AddFrameDataToProto(pb_tracker::Frame *pbFrameData, FrameData &fData)
Add frame tracked data into protobuf message.
FrameData GetTrackedData(size_t frameId)
Get tracked data for a given frame.
cv::Ptr< OPENCV_TRACKER_TYPE > selectTracker(std::string trackerType)
Definition CVTracker.cpp:37
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition ClipBase.h:88
This class represents a clip (used to arrange readers on the timeline)
Definition Clip.h:89
void Open() override
Open the internal reader.
Definition Clip.cpp:320
float End() const override
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve.
Definition Clip.cpp:356
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition Clip.cpp:391
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition Clip.cpp:274
Exception for invalid JSON.
Definition Exceptions.h:218
This namespace is the default namespace for all code in the openshot library.
Definition Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition Json.cpp:16