OpenShot Library | libopenshot 0.3.3
Loading...
Searching...
No Matches
CVStabilization.cpp
Go to the documentation of this file.
1
10// Copyright (c) 2008-2019 OpenShot Studios, LLC
11//
12// SPDX-License-Identifier: LGPL-3.0-or-later
13
14#include <fstream>
15#include <iomanip>
16#include <iostream>
17
18#include "CVStabilization.h"
19#include "Exceptions.h"
20
21#include "stabilizedata.pb.h"
22#include <google/protobuf/util/time_util.h>
23
24using namespace std;
25using namespace openshot;
26using google::protobuf::util::TimeUtil;
27
28// Set default smoothing window value to compute stabilization
29CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController)
30: processingController(&processingController){
31 SetJson(processInfoJson);
32 start = 1;
33 end = 1;
34}
35
36// Process clip and store necessary stabilization data
37void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
38
39 if(error){
40 return;
41 }
42 processingController->SetError(false, "");
43
44 start = _start; end = _end;
45 // Compute max and average transformation parameters
46 avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0;
47
48 video.Open();
49 // Save original video width and height
50 cv::Size readerDims(video.Reader()->info.width, video.Reader()->info.height);
51
52 size_t frame_number;
53 if(!process_interval || end <= 1 || end-start == 0){
54 // Get total number of frames in video
55 start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
56 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
57 }
58
59 // Extract and track opticalflow features for each frame
60 for (frame_number = start; frame_number <= end; frame_number++)
61 {
62 // Stop the feature tracker process
63 if(processingController->ShouldStop()){
64 return;
65 }
66
67 std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
68
69 // Grab OpenCV Mat image
70 cv::Mat cvimage = f->GetImageCV();
71 // Resize frame to original video width and height if they differ
72 if(cvimage.size().width != readerDims.width || cvimage.size().height != readerDims.height)
73 cv::resize(cvimage, cvimage, cv::Size(readerDims.width, readerDims.height));
74 cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
75
76 if(!TrackFrameFeatures(cvimage, frame_number)){
77 prev_to_cur_transform.push_back(TransformParam(0, 0, 0));
78 }
79
80 // Update progress
81 processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
82 }
83
84 // Calculate trajectory data
85 std::vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
86
87 // Calculate and save smoothed trajectory data
88 trajectoryData = SmoothTrajectory(trajectory);
89
90 // Calculate and save transformation data
91 transformationData = GenNewCamPosition(trajectoryData);
92
93 // Normalize smoothed trajectory data
94 for(auto &dataToNormalize : trajectoryData){
95 dataToNormalize.second.x/=readerDims.width;
96 dataToNormalize.second.y/=readerDims.height;
97 }
98 // Normalize transformation data
99 for(auto &dataToNormalize : transformationData){
100 dataToNormalize.second.dx/=readerDims.width;
101 dataToNormalize.second.dy/=readerDims.height;
102 }
103}
104
105// Track current frame features and find the relative transformation
106bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
107 // Check if there are black frames
108 if(cv::countNonZero(frame) < 1){
109 return false;
110 }
111
112 // Initialize prev_grey if not
113 if(prev_grey.empty()){
114 prev_grey = frame;
115 return true;
116 }
117
118 // OpticalFlow features vector
119 std::vector <cv::Point2f> prev_corner, cur_corner;
120 std::vector <cv::Point2f> prev_corner2, cur_corner2;
121 std::vector <uchar> status;
122 std::vector <float> err;
123 // Extract new image features
124 cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
125 // Track features
126 cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err);
127 // Remove untracked features
128 for(size_t i=0; i < status.size(); i++) {
129 if(status[i]) {
130 prev_corner2.push_back(prev_corner[i]);
131 cur_corner2.push_back(cur_corner[i]);
132 }
133 }
134 // In case no feature was detected
135 if(prev_corner2.empty() || cur_corner2.empty()){
136 last_T = cv::Mat();
137 // prev_grey = cv::Mat();
138 return false;
139 }
140
141 // Translation + rotation only
142 cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
143
144 double da, dx, dy;
145 // If T has nothing inside return (probably a segment where there is nothing to stabilize)
146 if(T.size().width == 0 || T.size().height == 0){
147 return false;
148 }
149 else{
150 // If no transformation is found, just use the last known good transform
151 if(T.data == NULL){
152 if(!last_T.empty())
153 last_T.copyTo(T);
154 else
155 return false;
156 }
157 // Decompose T
158 dx = T.at<double>(0,2);
159 dy = T.at<double>(1,2);
160 da = atan2(T.at<double>(1,0), T.at<double>(0,0));
161 }
162
163 // Filter transformations parameters, if they are higher than these: return
164 if(dx > 200 || dy > 200 || da > 0.1){
165 return false;
166 }
167
168 // Keep computing average and max transformation parameters
169 avr_dx+=fabs(dx);
170 avr_dy+=fabs(dy);
171 avr_da+=fabs(da);
172 if(fabs(dx) > max_dx)
173 max_dx = dx;
174 if(fabs(dy) > max_dy)
175 max_dy = dy;
176 if(fabs(da) > max_da)
177 max_da = da;
178
179 T.copyTo(last_T);
180
181 prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
182 frame.copyTo(prev_grey);
183
184 return true;
185}
186
187std::vector<CamTrajectory> CVStabilization::ComputeFramesTrajectory(){
188
189 // Accumulated frame to frame transform
190 double a = 0;
191 double x = 0;
192 double y = 0;
193
194 vector <CamTrajectory> trajectory; // trajectory at all frames
195
196 // Compute global camera trajectory. First frame is the origin
197 for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
198 x += prev_to_cur_transform[i].dx;
199 y += prev_to_cur_transform[i].dy;
200 a += prev_to_cur_transform[i].da;
201
202 // Save trajectory data to vector
203 trajectory.push_back(CamTrajectory(x,y,a));
204 }
205 return trajectory;
206}
207
208std::map<size_t,CamTrajectory> CVStabilization::SmoothTrajectory(std::vector <CamTrajectory> &trajectory){
209
210 std::map <size_t,CamTrajectory> smoothed_trajectory; // trajectory at all frames
211
212 for(size_t i=0; i < trajectory.size(); i++) {
213 double sum_x = 0;
214 double sum_y = 0;
215 double sum_a = 0;
216 int count = 0;
217
218 for(int j=-smoothingWindow; j <= smoothingWindow; j++) {
219 if(i+j < trajectory.size()) {
220 sum_x += trajectory[i+j].x;
221 sum_y += trajectory[i+j].y;
222 sum_a += trajectory[i+j].a;
223
224 count++;
225 }
226 }
227
228 double avg_a = sum_a / count;
229 double avg_x = sum_x / count;
230 double avg_y = sum_y / count;
231
232 // Add smoothed trajectory data to map
233 smoothed_trajectory[i + start] = CamTrajectory(avg_x, avg_y, avg_a);
234 }
235 return smoothed_trajectory;
236}
237
238// Generate new transformations parameters for each frame to follow the smoothed trajectory
239std::map<size_t,TransformParam> CVStabilization::GenNewCamPosition(std::map <size_t,CamTrajectory> &smoothed_trajectory){
240 std::map <size_t,TransformParam> new_prev_to_cur_transform;
241
242 // Accumulated frame to frame transform
243 double a = 0;
244 double x = 0;
245 double y = 0;
246
247 for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
248 x += prev_to_cur_transform[i].dx;
249 y += prev_to_cur_transform[i].dy;
250 a += prev_to_cur_transform[i].da;
251
252 // target - current
253 double diff_x = smoothed_trajectory[i + start].x - x;
254 double diff_y = smoothed_trajectory[i + start].y - y;
255 double diff_a = smoothed_trajectory[i + start].a - a;
256
257 double dx = prev_to_cur_transform[i].dx + diff_x;
258 double dy = prev_to_cur_transform[i].dy + diff_y;
259 double da = prev_to_cur_transform[i].da + diff_a;
260
261 // Add transformation data to map
262 new_prev_to_cur_transform[i + start] = TransformParam(dx, dy, da);
263 }
264 return new_prev_to_cur_transform;
265}
266
267// Save stabilization data to protobuf file
269 using std::ios;
270
271 // Create stabilization message
272 pb_stabilize::Stabilization stabilizationMessage;
273
274 std::map<size_t,CamTrajectory>::iterator trajData = trajectoryData.begin();
275 std::map<size_t,TransformParam>::iterator transData = transformationData.begin();
276
277 // Iterate over all frames data and save in protobuf message
278 for(; trajData != trajectoryData.end(); ++trajData, ++transData){
279 AddFrameDataToProto(stabilizationMessage.add_frame(), trajData->second, transData->second, trajData->first);
280 }
281 // Add timestamp
282 *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
283
284 // Write the new message to disk.
285 std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
286 if (!stabilizationMessage.SerializeToOstream(&output)) {
287 std::cerr << "Failed to write protobuf message." << std::endl;
288 return false;
289 }
290
291 // Delete all global objects allocated by libprotobuf.
292 google::protobuf::ShutdownProtobufLibrary();
293
294 return true;
295}
296
297// Add frame stabilization data into protobuf message
298void CVStabilization::AddFrameDataToProto(pb_stabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number){
299
300 // Save frame number
301 pbFrameData->set_id(frame_number);
302
303 // Save camera trajectory data
304 pbFrameData->set_a(trajData.a);
305 pbFrameData->set_x(trajData.x);
306 pbFrameData->set_y(trajData.y);
307
308 // Save transformation data
309 pbFrameData->set_da(transData.da);
310 pbFrameData->set_dx(transData.dx);
311 pbFrameData->set_dy(transData.dy);
312}
313
315
316 // Check if the stabilizer info for the requested frame exists
317 if ( transformationData.find(frameId) == transformationData.end() ) {
318
319 return TransformParam();
320 } else {
321
322 return transformationData[frameId];
323 }
324}
325
327
328 // Check if the stabilizer info for the requested frame exists
329 if ( trajectoryData.find(frameId) == trajectoryData.end() ) {
330
331 return CamTrajectory();
332 } else {
333
334 return trajectoryData[frameId];
335 }
336}
337
338// Load JSON string into this object
339void CVStabilization::SetJson(const std::string value) {
340 // Parse JSON string into JSON objects
341 try
342 {
343 const Json::Value root = openshot::stringToJson(value);
344 // Set all values that match
345
346 SetJsonValue(root);
347 }
348 catch (const std::exception& e)
349 {
350 // Error parsing JSON (or missing keys)
351 throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
352 }
353}
354
355// Load Json::Value into this object
356void CVStabilization::SetJsonValue(const Json::Value root) {
357
358 // Set data from Json (if key is found)
359 if (!root["protobuf_data_path"].isNull()){
360 protobuf_data_path = (root["protobuf_data_path"].asString());
361 }
362 if (!root["smoothing-window"].isNull()){
363 smoothingWindow = (root["smoothing-window"].asInt());
364 }
365}
366
367/*
368||||||||||||||||||||||||||||||||||||||||||||||||||
369 ONLY FOR MAKE TEST
370||||||||||||||||||||||||||||||||||||||||||||||||||
371*/
372
373// Load protobuf data file
375 using std::ios;
376 // Create stabilization message
377 pb_stabilize::Stabilization stabilizationMessage;
378 // Read the existing tracker message.
379 std::fstream input(protobuf_data_path, ios::in | ios::binary);
380 if (!stabilizationMessage.ParseFromIstream(&input)) {
381 std::cerr << "Failed to parse protobuf message." << std::endl;
382 return false;
383 }
384
385 // Make sure the data maps are empty
386 transformationData.clear();
387 trajectoryData.clear();
388
389 // Iterate over all frames of the saved message and assign to the data maps
390 for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) {
391 const pb_stabilize::Frame& pbFrameData = stabilizationMessage.frame(i);
392
393 // Load frame number
394 size_t id = pbFrameData.id();
395
396 // Load camera trajectory data
397 float x = pbFrameData.x();
398 float y = pbFrameData.y();
399 float a = pbFrameData.a();
400
401 // Assign data to trajectory map
402 trajectoryData[id] = CamTrajectory(x,y,a);
403
404 // Load transformation data
405 float dx = pbFrameData.dx();
406 float dy = pbFrameData.dy();
407 float da = pbFrameData.da();
408
409 // Assing data to transformation map
410 transformationData[id] = TransformParam(dx,dy,da);
411 }
412
413 // Delete all global objects allocated by libprotobuf.
414 google::protobuf::ShutdownProtobufLibrary();
415
416 return true;
417}
Header file for CVStabilization class.
Header file for all Exception classes.
TransformParam GetTransformParamData(size_t frameId)
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
CVStabilization(std::string processInfoJson, ProcessingController &processingController)
Set default smoothing window value to compute stabilization.
void AddFrameDataToProto(pb_stabilize::Frame *pbFrameData, CamTrajectory &trajData, TransformParam &transData, size_t frame_number)
Add frame stabilization data into protobuf message.
std::map< size_t, TransformParam > transformationData
void stabilizeClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Process clip and store necessary stabilization data.
std::map< size_t, CamTrajectory > trajectoryData
CamTrajectory GetCamTrajectoryTrackedData(size_t frameId)
void SetJson(const std::string value)
Load JSON string into this object.
void SetError(bool err, std::string message)
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition ClipBase.h:88
This class represents a clip (used to arrange readers on the timeline)
Definition Clip.h:89
void Open() override
Open the internal reader.
Definition Clip.cpp:320
float End() const override
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve.
Definition Clip.cpp:356
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition Clip.cpp:391
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition Clip.cpp:274
Exception for invalid JSON.
Definition Exceptions.h:218
This namespace is the default namespace for all code in the openshot library.
Definition Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition Json.cpp:16