21#include "objdetectdata.pb.h"
34display_box_text(1.0), display_boxes(1.0)
37 init_effect_details();
48 display_box_text(1.0), display_boxes(1.0)
51 init_effect_details();
58void ObjectDetection::init_effect_details()
76 std::shared_ptr<QImage> frame_image = frame->GetImage();
79 if(!frame_image || frame_image->isNull()) {
83 QPainter painter(frame_image.get());
84 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform);
86 if (detectionsData.find(frame_number) != detectionsData.end()) {
88 for (
int i = 0; i < detections.
boxes.size(); i++) {
89 if (detections.
confidences.at(i) < confidence_threshold ||
90 (!display_classes.empty() &&
91 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end())) {
95 int objectId = detections.
objectIds.at(i);
99 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
102 if (parentClip && trackedObject->Contains(frame_number) && trackedObject->visible.GetValue(frame_number) == 1) {
103 BBox trackedBox = trackedObject->GetBox(frame_number);
104 QRectF boxRect((trackedBox.
cx - trackedBox.
width / 2) * frame_image->width(),
105 (trackedBox.
cy - trackedBox.
height / 2) * frame_image->height(),
106 trackedBox.
width * frame_image->width(),
107 trackedBox.
height * frame_image->height());
110 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
111 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
112 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
113 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
114 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
115 float bg_corner = trackedObject->background_corner.GetValue(frame_number);
118 QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha));
119 pen.setWidth(stroke_width);
123 QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha));
124 painter.setBrush(brush);
126 if (display_boxes.
GetValue(frame_number) == 1 && trackedObject->draw_box.GetValue(frame_number) == 1) {
128 painter.drawRoundedRect(boxRect, bg_corner, bg_corner);
131 if(display_box_text.
GetValue(frame_number) == 1) {
134 int classId = detections.
classIds.at(i);
137 QString label = QString::number(objectId);
138 if (!classNames.empty()) {
139 label = QString::fromStdString(classNames[classId]) +
":" + label;
144 font.setPixelSize(14);
145 painter.setFont(font);
148 QFontMetrics fontMetrics(font);
149 QSize labelSize = fontMetrics.size(Qt::TextSingleLine, label);
152 double left = boxRect.center().x() - (labelSize.width() / 2.0);
153 double top = std::max(
static_cast<int>(boxRect.top()), labelSize.height()) - 4.0;
156 painter.drawText(QPointF(left, top), label);
172 pb_objdetect::ObjDetect objMessage;
175 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
176 if (!objMessage.ParseFromIstream(&input)) {
177 std::cerr <<
"Failed to parse protobuf message." << std::endl;
183 detectionsData.clear();
189 for(
int i = 0; i < objMessage.classnames_size(); i++)
191 classNames.push_back(objMessage.classnames(i));
192 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
196 for (
size_t i = 0; i < objMessage.frame_size(); i++)
199 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
202 size_t id = pbFrameData.id();
205 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
208 std::vector<int> classIds;
209 std::vector<float> confidences;
210 std::vector<cv::Rect_<float>> boxes;
211 std::vector<int> objectIds;
214 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
217 float x = pBox.Get(i).x();
218 float y = pBox.Get(i).y();
219 float w = pBox.Get(i).w();
220 float h = pBox.Get(i).h();
222 int classId = pBox.Get(i).classid();
224 float confidence = pBox.Get(i).confidence();
227 int objectId = pBox.Get(i).objectid();
235 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
240 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
242 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
244 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
246 trackedObjPtr->ParentClip(parentClip);
250 trackedObjPtr->Id(std::to_string(objectId));
255 cv::Rect_<float> box(x, y, w, h);
258 boxes.push_back(box);
259 classIds.push_back(classId);
260 confidences.push_back(confidence);
261 objectIds.push_back(objectId);
265 detectionsData[
id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
269 google::protobuf::ShutdownProtobufLibrary();
279 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
280 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
281 root[
"visible_class_names"] = Json::Value(Json::arrayValue);
284 if (detectionsData.find(frame_number) == detectionsData.end()){
285 return root.toStyledString();
290 for(
int i = 0; i<detections.
boxes.size(); i++){
292 if(detections.
confidences.at(i) < confidence_threshold){
297 auto className = classNames[detections.
classIds.at(i)];
300 if (!display_classes.empty()) {
301 auto it = std::find(display_classes.begin(), display_classes.end(), className);
302 if (it == display_classes.end()) {
306 root[
"visible_class_names"].append(className);
309 root[
"visible_class_names"].append(className);
312 int objectId = detections.
objectIds.at(i);
317 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
319 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
320 trackedObject->second->ExactlyContains(frame_number)){
322 root[
"visible_objects_index"].append(trackedObject->first);
323 root[
"visible_objects_id"].append(trackedObject->second->Id());
327 return root.toStyledString();
343 root[
"protobuf_data_path"] = protobuf_data_path;
345 root[
"confidence_threshold"] = confidence_threshold;
346 root[
"display_box_text"] = display_box_text.
JsonValue();
347 root[
"display_boxes"] = display_boxes.
JsonValue();
352 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
354 objects[trackedObject.second->Id()] = trackedObjectJSON;
356 root[
"objects"] = objects;
372 catch (
const std::exception& e)
375 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
385 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
386 protobuf_data_path = root[
"protobuf_data_path"].asString();
389 throw InvalidFile(
"Invalid protobuf data path",
"");
390 protobuf_data_path =
"";
395 if (!root[
"selected_object_index"].isNull())
398 if (!root[
"confidence_threshold"].isNull())
399 confidence_threshold = root[
"confidence_threshold"].asFloat();
401 if (!root[
"display_box_text"].isNull())
402 display_box_text.
SetJsonValue(root[
"display_box_text"]);
404 if (!root[
"display_boxes"].isNull())
407 if (!root[
"class_filter"].isNull()) {
408 class_filter = root[
"class_filter"].asString();
411 QString qClassFilter = QString::fromStdString(root[
"class_filter"].asString());
414 QStringList classList = qClassFilter.split(
',', QString::SkipEmptyParts);
415 display_classes.clear();
418 for (
const QString &classItem : classList) {
419 QString trimmedItem = classItem.trimmed().toLower();
420 if (!trimmedItem.isEmpty()) {
421 display_classes.push_back(trimmedItem.toStdString());
426 if (!root[
"objects"].isNull()){
428 std::string obj_id = std::to_string(trackedObject.first);
429 if(!root[
"objects"][obj_id].isNull()){
430 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
436 if (!root[
"objects_id"].isNull()){
438 Json::Value trackedObjectJSON;
439 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
440 trackedObject.second->SetJsonValue(trackedObjectJSON);
455 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
457 objects[selectedObject->Id()] = trackedObjectJSON;
460 root[
"objects"] = objects;
463 root[
"confidence_threshold"] =
add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
464 root[
"class_filter"] =
add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
466 root[
"display_box_text"] =
add_property_json(
"Draw All Text", display_box_text.
GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1,
false, requested_frame);
470 root[
"display_boxes"] =
add_property_json(
"Draw All Boxes", display_boxes.
GetValue(requested_frame),
"int",
"", &display_boxes, 0, 1,
false, requested_frame);
475 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
std::string id
ID Property for all derived Clip and Effect classes.
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
This class represents a clip (used to arrange readers on the timeline)
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
Json::Value BasePropertiesJSON(int64_t requested_frame) const
Generate JSON object of base properties (recommended to be used by all effects)
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
EffectInfoStruct info
Information about the current effect.
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Exception for files that can not be found or opened.
Exception for invalid JSON.
A Keyframe is a collection of Point instances, which is used to vary a number or property over time.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
double GetValue(int64_t index) const
Get the value at a specific index.
Json::Value JsonValue() const
Generate Json::Value for this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
int selectedObjectIndex
Index of the Tracked Object that was selected to modify it's properties.
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
This method is required for all derived classes of EffectBase, and returns a modified openshot::Frame...
ObjectDetection()
Default constructor.
bool LoadObjDetectdData(std::string inputFilePath)
Load protobuf data file.
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
std::string Json() const override
Generate JSON string of this object.
std::string PropertiesJSON(int64_t requested_frame) const override
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
void SetJson(const std::string value) override
Load JSON string into this object.
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
Keyframe stroke_alpha
Stroke box opacity.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
bool has_video
Determines if this effect manipulates the image of a frame.
bool has_audio
Determines if this effect manipulates the audio of a frame.
std::string class_name
The class name of the effect.
std::string name
The name of the effect.
std::string description
The description of this effect and what it does.
bool has_tracked_object
Determines if this effect track objects through the clip.