OpenShot Library | libopenshot 0.3.3
Loading...
Searching...
No Matches
Frame.cpp
Go to the documentation of this file.
1
9// Copyright (c) 2008-2019 OpenShot Studios, LLC
10//
11// SPDX-License-Identifier: LGPL-3.0-or-later
12
13#include <thread> // for std::this_thread::sleep_for
14#include <chrono> // for std::chrono::milliseconds
15#include <sstream>
16#include <iomanip>
17
18#include "Frame.h"
19#include "AudioBufferSource.h"
20#include "AudioResampler.h"
21#include "QtUtilities.h"
22
23#include <AppConfig.h>
24#include <juce_audio_basics/juce_audio_basics.h>
25#include <juce_audio_devices/juce_audio_devices.h>
26
27#include <QApplication>
28#include <QImage>
29#include <QPixmap>
30#include <QBitmap>
31#include <QColor>
32#include <QString>
33#include <QVector>
34#include <QPainter>
35#include <QHBoxLayout>
36#include <QWidget>
37#include <QLabel>
38#include <QPointF>
39#include <QWidget>
40
41using namespace std;
42using namespace openshot;
43
44// Constructor - image & audio
45Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
46 : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
47 number(number), width(width), height(height),
48 pixel_ratio(1,1), color(color),
49 channels(channels), channel_layout(LAYOUT_STEREO),
50 sample_rate(44100),
51 has_audio_data(false), has_image_data(false),
52 max_audio_sample(0)
53{
54 // zero (fill with silence) the audio buffer
55 audio->clear();
56}
57
58// Delegating Constructor - blank frame
59Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
60
61// Delegating Constructor - image only
62Frame::Frame(int64_t number, int width, int height, std::string color)
63 : Frame::Frame(number, width, height, color, 0, 2) {}
64
65// Delegating Constructor - audio only
66Frame::Frame(int64_t number, int samples, int channels)
67 : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
68
69
70// Copy constructor
71Frame::Frame ( const Frame &other )
72{
73 // copy pointers and data
74 DeepCopy(other);
75}
76
77// Assignment operator
79{
80 // copy pointers and data
81 DeepCopy(other);
82
83 return *this;
84}
85
86// Copy data and pointers from another Frame instance
87void Frame::DeepCopy(const Frame& other)
88{
89 number = other.number;
90 channels = other.channels;
91 width = other.width;
92 height = other.height;
93 channel_layout = other.channel_layout;
96 sample_rate = other.sample_rate;
97 pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
98 color = other.color;
99 max_audio_sample = other.max_audio_sample;
100
101 if (other.image)
102 image = std::make_shared<QImage>(*(other.image));
103 if (other.audio)
104 audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
105 if (other.wave_image)
106 wave_image = std::make_shared<QImage>(*(other.wave_image));
107}
108
109// Destructor
111 // Clear all pointers
112 image.reset();
113 audio.reset();
114 #ifdef USE_OPENCV
115 imagecv.release();
116 #endif
117}
118
119// Display the frame image to the screen (primarily used for debugging reasons)
121{
122 if (!QApplication::instance()) {
123 // Only create the QApplication once
124 static int argc = 1;
125 static char* argv[1] = {NULL};
126 previewApp = std::make_shared<QApplication>(argc, argv);
127 }
128
129 // Get preview image
130 std::shared_ptr<QImage> previewImage = GetImage();
131
132 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
133 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
134 {
135 // Resize to fix DAR
136 previewImage = std::make_shared<QImage>(previewImage->scaled(
137 previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
138 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
139 }
140
141 // Create window
142 QWidget previewWindow;
143 previewWindow.setStyleSheet("background-color: #000000;");
144 QHBoxLayout layout;
145
146 // Create label with current frame's image
147 QLabel previewLabel;
148 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150 layout.addWidget(&previewLabel);
151
152 // Show the window
153 previewWindow.setLayout(&layout);
154 previewWindow.show();
155 previewApp->exec();
156}
157
158// Get an audio waveform image
159std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
160{
161 // Clear any existing waveform image
163
164 // Init a list of lines
165 QVector<QPointF> lines;
166 QVector<QPointF> labels;
167
168 // Calculate width of an image based on the # of samples
169 int total_samples = GetAudioSamplesCount();
170 if (total_samples > 0)
171 {
172 // If samples are present...
173 int new_height = 200 * audio->getNumChannels();
174 int height_padding = 20 * (audio->getNumChannels() - 1);
175 int total_height = new_height + height_padding;
176 int total_width = 0;
177 float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
178
179 // Loop through each audio channel
180 float Y = 100.0;
181 for (int channel = 0; channel < audio->getNumChannels(); channel++)
182 {
183 float X = 0.0;
184
185 // Get audio for this channel
186 const float *samples = audio->getReadPointer(channel);
187
188 for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
189 {
190 // Sample value (scaled to -100 to 100)
191 float value = samples[sample] * 100.0;
192
193 // Set threshold near zero (so we don't allow near-zero values)
194 // This prevents empty gaps from appearing in the waveform
195 if (value > -zero_height && value < 0.0) {
196 value = -zero_height;
197 } else if (value > 0.0 && value < zero_height) {
198 value = zero_height;
199 }
200
201 // Append a line segment for each sample
202 lines.push_back(QPointF(X, Y));
203 lines.push_back(QPointF(X, Y - value));
204 }
205
206 // Add Channel Label Coordinate
207 labels.push_back(QPointF(5.0, Y - 5.0));
208
209 // Increment Y
210 Y += (200 + height_padding);
211 total_width = X;
212 }
213
214 // Create blank image
215 wave_image = std::make_shared<QImage>(
216 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217 wave_image->fill(QColor(0,0,0,0));
218
219 // Load QPainter with wave_image device
220 QPainter painter(wave_image.get());
221
222 // Set pen color
223 QPen pen;
224 pen.setColor(QColor(Red, Green, Blue, Alpha));
225 pen.setWidthF(1.0);
226 pen.setStyle(Qt::SolidLine);
227 painter.setPen(pen);
228
229 // Draw the waveform
230 painter.drawLines(lines);
231 painter.end();
232 }
233 else
234 {
235 // No audio samples present
236 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237 wave_image->fill(QColor(QString::fromStdString("#000000")));
238 }
239
240 // Resize Image (if needed)
241 if (wave_image->width() != width || wave_image->height() != height) {
242 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243 wave_image = std::make_shared<QImage>(scaled_wave_image);
244 }
245
246 // Return new image
247 return wave_image;
248}
249
250// Clear the waveform image (and deallocate its memory)
252{
253 if (wave_image)
254 wave_image.reset();
255}
256
257// Get an audio waveform image pixels
258const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
259{
260 // Get audio wave form image
261 wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
262
263 // Return array of pixel packets
264 return wave_image->constBits();
265}
266
267// Display the wave form
269{
270 // Get audio wave form image
271 GetWaveform(720, 480, 0, 123, 255, 255);
272
273 if (!QApplication::instance()) {
274 // Only create the QApplication once
275 static int argc = 1;
276 static char* argv[1] = {NULL};
277 previewApp = std::make_shared<QApplication>(argc, argv);
278 }
279
280 // Create window
281 QWidget previewWindow;
282 previewWindow.setStyleSheet("background-color: #000000;");
283 QHBoxLayout layout;
284
285 // Create label with current frame's waveform image
286 QLabel previewLabel;
287 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289 layout.addWidget(&previewLabel);
290
291 // Show the window
292 previewWindow.setLayout(&layout);
293 previewWindow.show();
294 previewApp->exec();
295
296 // Deallocate waveform image
298}
299
300// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
301float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
302{
303 if (channel > 0) {
304 // return average magnitude for a specific channel/sample range
305 return audio->getMagnitude(channel, sample, magnitude_range);
306
307 } else {
308 // Return average magnitude for all channels
309 return audio->getMagnitude(sample, magnitude_range);
310 }
311}
312
313// Get an array of sample data (and optional reverse the sample values)
314float* Frame::GetAudioSamples(int channel) {
315
316 // Copy audio data
317 juce::AudioBuffer<float> *buffer(audio.get());
318
319 // return JUCE audio data for this channel
320 return buffer->getWritePointer(channel);
321}
322
323// Get an array of sample data (all channels interleaved together), using any sample rate
324float* Frame::GetInterleavedAudioSamples(int* sample_count)
325{
326 // Copy audio data
327 juce::AudioBuffer<float> *buffer(audio.get());
328
329 float *output = NULL;
330 int num_of_channels = audio->getNumChannels();
331 int num_of_samples = GetAudioSamplesCount();
332
333 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
334 output = new float[num_of_channels * num_of_samples];
335 int position = 0;
336
337 // Loop through samples in each channel (combining them)
338 for (int sample = 0; sample < num_of_samples; sample++)
339 {
340 for (int channel = 0; channel < num_of_channels; channel++)
341 {
342 // Add sample to output array
343 output[position] = buffer->getReadPointer(channel)[sample];
344
345 // increment position
346 position++;
347 }
348 }
349
350 // Update sample count (since it might have changed due to resampling)
351 *sample_count = num_of_samples;
352
353 // return combined array
354 return output;
355}
356
357// Get number of audio channels
359{
360 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
361 if (audio)
362 return audio->getNumChannels();
363 else
364 return 0;
365}
366
367// Get number of audio samples
369{
370 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371 return max_audio_sample;
372}
373
378
379// Get the size in bytes of this frame (rough estimate)
381{
382 int64_t total_bytes = 0;
383 if (image) {
384 total_bytes += static_cast<int64_t>(
385 width * height * sizeof(char) * 4);
386 }
387 if (audio) {
388 // approximate audio size (sample rate / 24 fps)
389 total_bytes += (sample_rate / 24.0) * sizeof(float);
390 }
391
392 // return size of this frame
393 return total_bytes;
394}
395
396// Get pixel data (as packets)
397const unsigned char* Frame::GetPixels()
398{
399 // Check for blank image
400 if (!image)
401 // Fill with black
402 AddColor(width, height, color);
403
404 // Return array of pixel packets
405 return image->constBits();
406}
407
408// Get pixel data (for only a single scan-line)
409const unsigned char* Frame::GetPixels(int row)
410{
411 // Check for blank image
412 if (!image)
413 // Fill with black
414 AddColor(width, height, color);
415
416 // Return array of pixel packets
417 return image->constScanLine(row);
418}
419
420// Check a specific pixel color value (returns True/False)
421bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
422 int col_pos = col * 4; // Find column array position
423 if (!image || row < 0 || row >= (height - 1) ||
424 col_pos < 0 || col_pos >= (width - 1) ) {
425 // invalid row / col
426 return false;
427 }
428 // Check pixel color
429 const unsigned char* pixels = GetPixels(row);
430 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
434 // Pixel color matches successfully
435 return true;
436 } else {
437 // Pixel color does not match
438 return false;
439 }
440}
441
442// Set Pixel Aspect Ratio
443void Frame::SetPixelRatio(int num, int den)
444{
445 pixel_ratio.num = num;
446 pixel_ratio.den = den;
447}
448
449// Set frame number
450void Frame::SetFrameNumber(int64_t new_number)
451{
452 number = new_number;
453}
454
455// Calculate the # of samples per video frame (for a specific frame number and frame rate)
456int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
457{
458 // Directly return 0 if there are no channels
459 // so that we do not need to deal with NaNs later
460 if (channels == 0) return 0;
461
462 // Get the total # of samples for the previous frame, and the current frame (rounded)
463 double fps_rate = fps.Reciprocal().ToDouble();
464
465 // Determine previous samples total, and make sure it's evenly divisible by the # of channels
466 double previous_samples = (sample_rate * fps_rate) * (number - 1);
467 double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
468 previous_samples -= previous_samples_remainder;
469
470 // Determine the current samples total, and make sure it's evenly divisible by the # of channels
471 double total_samples = (sample_rate * fps_rate) * number;
472 double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
473 total_samples -= total_samples_remainder;
474
475 // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
476 // be evenly divided into frames, so each frame can have have different # of samples.
477 int samples_per_frame = round(total_samples - previous_samples);
478 if (samples_per_frame < 0)
479 samples_per_frame = 0;
480 return samples_per_frame;
481}
482
483// Calculate the # of samples per video frame (for the current frame number)
484int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
485{
486 return GetSamplesPerFrame(number, fps, sample_rate, channels);
487}
488
489// Get height of image
491{
492 return height;
493}
494
495// Get height of image
497{
498 return width;
499}
500
501// Get the original sample rate of this frame's audio data
503{
504 return sample_rate;
505}
506
507// Get the original sample rate of this frame's audio data
509{
510 return channel_layout;
511}
512
513
514// Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
515void Frame::Save(std::string path, float scale, std::string format, int quality)
516{
517 // Get preview image
518 std::shared_ptr<QImage> previewImage = GetImage();
519
520 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
521 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
522 {
523 // Resize to fix DAR
524 previewImage = std::make_shared<QImage>(previewImage->scaled(
525 previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
526 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
527 }
528
529 // scale image if needed
530 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
531 {
532 // Resize image
533 previewImage = std::make_shared<QImage>(previewImage->scaled(
534 previewImage->size().width() * scale, previewImage->size().height() * scale,
535 Qt::KeepAspectRatio, Qt::SmoothTransformation));
536 }
537
538 // Save image
539 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
540}
541
542// Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
543void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
544 std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
545
546 // Create blank thumbnail image & fill background color
547 auto thumbnail = std::make_shared<QImage>(
548 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
549 thumbnail->fill(QColor(QString::fromStdString(background_color)));
550
551 // Create painter
552 QPainter painter(thumbnail.get());
553 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
554
555 // Get preview image
556 std::shared_ptr<QImage> previewImage = GetImage();
557
558 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
559 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
560 {
561 // Calculate correct DAR (display aspect ratio)
562 int aspect_width = previewImage->size().width();
563 int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
564
565 // Resize to fix DAR
566 previewImage = std::make_shared<QImage>(previewImage->scaled(
567 aspect_width, aspect_height,
568 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
569 }
570
571 // Resize frame image
572 if (ignore_aspect)
573 // Ignore aspect ratio
574 previewImage = std::make_shared<QImage>(previewImage->scaled(
575 new_width, new_height,
576 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
577 else
578 // Maintain aspect ratio
579 previewImage = std::make_shared<QImage>(previewImage->scaled(
580 new_width, new_height,
581 Qt::KeepAspectRatio, Qt::SmoothTransformation));
582
583 // Composite frame image onto background (centered)
584 int x = (new_width - previewImage->size().width()) / 2.0; // center
585 int y = (new_height - previewImage->size().height()) / 2.0; // center
586 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
587
588
589 // Create transform and rotate (if needed)
590 QTransform transform;
591 float origin_x = previewImage->width() / 2.0;
592 float origin_y = previewImage->height() / 2.0;
593 transform.translate(origin_x, origin_y);
594 transform.rotate(rotate);
595 transform.translate(-origin_x,-origin_y);
596 painter.setTransform(transform);
597
598 // Draw image onto QImage
599 painter.drawImage(x, y, *previewImage);
600
601
602 // Overlay Image (if any)
603 if (overlay_path != "") {
604 // Open overlay
605 auto overlay = std::make_shared<QImage>();
606 overlay->load(QString::fromStdString(overlay_path));
607
608 // Set pixel format
609 overlay = std::make_shared<QImage>(
610 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
611
612 // Resize to fit
613 overlay = std::make_shared<QImage>(overlay->scaled(
614 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
615
616 // Composite onto thumbnail
617 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
618 painter.drawImage(0, 0, *overlay);
619 }
620
621
622 // Mask Image (if any)
623 if (mask_path != "") {
624 // Open mask
625 auto mask = std::make_shared<QImage>();
626 mask->load(QString::fromStdString(mask_path));
627
628 // Set pixel format
629 mask = std::make_shared<QImage>(
630 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
631
632 // Resize to fit
633 mask = std::make_shared<QImage>(mask->scaled(
634 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
635
636 // Negate mask
637 mask->invertPixels();
638
639 // Get pixels
640 unsigned char *pixels = static_cast<unsigned char *>(thumbnail->bits());
641 const unsigned char *mask_pixels = static_cast<const unsigned char *>(mask->constBits());
642
643 // Convert the mask image to grayscale
644 // Loop through pixels
645 for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
646 {
647 // Get the RGB values from the pixel
648 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
649 int Frame_Alpha = pixels[byte_index + 3];
650 int Mask_Value = constrain(Frame_Alpha - gray_value);
651
652 // Set all alpha pixels to gray value
653 pixels[byte_index + 3] = Mask_Value;
654 }
655 }
656
657
658 // End painter
659 painter.end();
660
661 // Save image
662 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
663}
664
665// Constrain a color value from 0 to 255
666int Frame::constrain(int color_value)
667{
668 // Constrain new color from 0 to 255
669 if (color_value < 0)
670 color_value = 0;
671 else if (color_value > 255)
672 color_value = 255;
673
674 return color_value;
675}
676
677void Frame::AddColor(int new_width, int new_height, std::string new_color)
678{
679 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
680 // Update parameters
681 width = new_width;
682 height = new_height;
683 color = new_color;
684 AddColor(QColor(QString::fromStdString(new_color)));
685}
686
687// Add (or replace) pixel data to the frame (based on a solid color)
688void Frame::AddColor(const QColor& new_color)
689{
690 // Create new image object, and fill with pixel data
691 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
692 image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
693
694 // Fill with solid color
695 image->fill(new_color);
696 has_image_data = true;
697}
698
699// Add (or replace) pixel data to the frame
701 int new_width, int new_height, int bytes_per_pixel,
702 QImage::Format type, const unsigned char *pixels_)
703{
704 if (has_image_data) {
705 // Delete the previous QImage
706 image.reset();
707 }
708
709 // Create new image object from pixel data
710 auto new_image = std::make_shared<QImage>(
711 pixels_,
712 new_width, new_height,
713 new_width * bytes_per_pixel,
714 type,
715 (QImageCleanupFunction) &openshot::cleanUpBuffer,
716 (void*) pixels_
717 );
718 AddImage(new_image);
719}
720
721// Add (or replace) pixel data to the frame
722void Frame::AddImage(std::shared_ptr<QImage> new_image)
723{
724 // Ignore blank images
725 if (!new_image)
726 return;
727
728 // assign image data
729 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
730 image = new_image;
731
732 // Always convert to Format_RGBA8888_Premultiplied (if different)
733 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
734 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
735
736 // Update height and width
737 width = image->width();
738 height = image->height();
739 has_image_data = true;
740}
741
742// Add (or replace) pixel data to the frame (for only the odd or even lines)
743void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
744{
745 // Ignore blank new_image
746 if (!new_image)
747 return;
748
749 // Check for blank source image
750 if (!image) {
751 // Replace the blank source image
752 AddImage(new_image);
753
754 } else {
755 // Ignore image of different sizes or formats
756 bool ret=false;
757 if (image == new_image || image->size() != new_image->size()) {
758 ret = true;
759 }
760 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
761 new_image = std::make_shared<QImage>(
762 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
763 }
764 if (ret) {
765 return;
766 }
767
768 // Get the frame's image
769 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
770 unsigned char *pixels = image->bits();
771 const unsigned char *new_pixels = new_image->constBits();
772
773 // Loop through the scanlines of the image (even or odd)
774 int start = 0;
775 if (only_odd_lines)
776 start = 1;
777
778 for (int row = start; row < image->height(); row += 2) {
779 int offset = row * image->bytesPerLine();
780 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
781 }
782
783 // Update height and width
784 height = image->height();
785 width = image->width();
786 has_image_data = true;
787 }
788}
789
790
791// Resize audio container to hold more (or less) samples and channels
792void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
793{
794 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
795
796 // Resize JUCE audio buffer
797 audio->setSize(channels, length, true, true, false);
798 channel_layout = layout;
799 sample_rate = rate;
800
801 // Calculate max audio sample added
802 max_audio_sample = length;
803}
804
805// Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times
806// you invoke this method)
808 if (audio && !audio_reversed) {
809 // Reverse audio buffer
810 audio->reverse(0, audio->getNumSamples());
811 audio_reversed = true;
812 }
813}
814
815// Add audio samples to a specific channel
816void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
817 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
818
819 // Clamp starting sample to 0
820 int destStartSampleAdjusted = max(destStartSample, 0);
821
822 // Extend audio container to hold more (or less) samples and channels.. if needed
823 int new_length = destStartSampleAdjusted + numSamples;
824 int new_channel_length = audio->getNumChannels();
825 if (destChannel >= new_channel_length)
826 new_channel_length = destChannel + 1;
827 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
828 audio->setSize(new_channel_length, new_length, true, true, false);
829
830 // Clear the range of samples first (if needed)
831 if (replaceSamples)
832 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
833
834 // Add samples to frame's audio buffer
835 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
836 has_audio_data = true;
837
838 // Calculate max audio sample added
839 if (new_length > max_audio_sample)
840 max_audio_sample = new_length;
841
842 // Reset audio reverse flag
843 audio_reversed = false;
844}
845
846// Apply gain ramp (i.e. fading volume)
847void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
848{
849 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
850
851 // Apply gain ramp
852 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
853}
854
855// Get pointer to Magick++ image object
856std::shared_ptr<QImage> Frame::GetImage()
857{
858 // Check for blank image
859 if (!image)
860 // Fill with black
861 AddColor(width, height, color);
862
863 return image;
864}
865
866#ifdef USE_OPENCV
867
868// Convert Qimage to Mat
869cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
870
871 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
872 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
873 int from_to[] = { 0,0, 1,1, 2,2 };
874 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
875 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
876 return mat2;
877}
878
879// Get pointer to OpenCV image object
881{
882 // Check for blank image
883 if (!image)
884 // Fill with black
885 AddColor(width, height, color);
886
887 // if (imagecv.empty())
888 // Convert Qimage to Mat
889 imagecv = Qimage2mat(image);
890
891 return imagecv;
892}
893
894std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
895 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
896 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
897
898 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
899
900 // Always convert to RGBA8888 (if different)
901 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
902 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
903
904 return imgIn;
905}
906
907// Set pointer to OpenCV image object
908void Frame::SetImageCV(cv::Mat _image)
909{
910 imagecv = _image;
911 image = Mat2Qimage(_image);
912}
913#endif
914
915// Play audio samples for this frame
917{
918 // Check if samples are present
920 return;
921
922 juce::AudioDeviceManager deviceManager;
923 juce::String error = deviceManager.initialise (
924 0, /* number of input channels */
925 2, /* number of output channels */
926 0, /* no XML settings.. */
927 true /* select default device on failure */);
928
929 // Output error (if any)
930 if (error.isNotEmpty()) {
931 cout << "Error on initialise(): " << error << endl;
932 }
933
934 juce::AudioSourcePlayer audioSourcePlayer;
935 deviceManager.addAudioCallback (&audioSourcePlayer);
936
937 std::unique_ptr<AudioBufferSource> my_source;
938 my_source.reset (new AudioBufferSource (audio.get()));
939
940 // Create TimeSliceThread for audio buffering
941 juce::TimeSliceThread my_thread("Audio buffer thread");
942
943 // Start thread
944 my_thread.startThread();
945
946 juce::AudioTransportSource transport1;
947 transport1.setSource (my_source.get(),
948 5000, // tells it to buffer this many samples ahead
949 &my_thread,
950 (double) sample_rate,
951 audio->getNumChannels()); // sample rate of source
952 transport1.setPosition (0);
953 transport1.setGain(1.0);
954
955
956 // Create MIXER
957 juce::MixerAudioSource mixer;
958 mixer.addInputSource(&transport1, false);
959 audioSourcePlayer.setSource (&mixer);
960
961 // Start transports
962 transport1.start();
963
964 while (transport1.isPlaying())
965 {
966 cout << "playing" << endl;
967 std::this_thread::sleep_for(std::chrono::seconds(1));
968 }
969
970 cout << "DONE!!!" << endl;
971
972 transport1.stop();
973 transport1.setSource (0);
974 audioSourcePlayer.setSource (0);
975 my_thread.stopThread(500);
976 deviceManager.removeAudioCallback (&audioSourcePlayer);
977 deviceManager.closeAudioDevice();
978 deviceManager.removeAllChangeListeners();
979 deviceManager.dispatchPendingMessages();
980
981 cout << "End of Play()" << endl;
982
983
984}
985
986// Add audio silence
987void Frame::AddAudioSilence(int numSamples)
988{
989 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
990
991 // Resize audio container
992 audio->setSize(channels, numSamples, false, true, false);
993 audio->clear();
994 has_audio_data = true;
995
996 // Calculate max audio sample added
997 max_audio_sample = numSamples;
998
999 // Reset audio reverse flag
1000 audio_reversed = false;
1001}
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
This class represents a fraction.
Definition Fraction.h:30
int num
Numerator for the fraction.
Definition Fraction.h:32
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition Fraction.cpp:40
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition Fraction.cpp:78
int den
Denominator for the fraction.
Definition Fraction.h:33
This class represents a single frame of video (i.e. image & audio data)
Definition Frame.h:91
Frame & operator=(const Frame &other)
Assignment operator.
Definition Frame.cpp:78
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition Frame.h:117
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition Frame.cpp:894
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition Frame.cpp:677
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition Frame.cpp:397
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition Frame.cpp:159
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition Frame.cpp:515
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition Frame.cpp:120
int GetAudioChannelsCount()
Get number of audio channels.
Definition Frame.cpp:358
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition Frame.cpp:869
bool has_image_data
This frame has been loaded with pixel data.
Definition Frame.h:120
int GetWidth()
Get height of image.
Definition Frame.cpp:496
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition Frame.cpp:502
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition Frame.cpp:792
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition Frame.cpp:87
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition Frame.cpp:880
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition Frame.cpp:251
void Play()
Play audio samples for this frame.
Definition Frame.cpp:916
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition Frame.cpp:324
bool has_audio_data
This frame has been loaded with audio data.
Definition Frame.h:119
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition Frame.cpp:380
openshot::ChannelLayout ChannelsLayout()
Definition Frame.cpp:508
void AddAudioSilence(int numSamples)
Add audio silence.
Definition Frame.cpp:987
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition Frame.cpp:543
void DisplayWaveform()
Display the wave form.
Definition Frame.cpp:268
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition Frame.cpp:421
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
Definition Frame.cpp:314
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition Frame.cpp:301
virtual ~Frame()
Destructor.
Definition Frame.cpp:110
void ReverseAudio()
Definition Frame.cpp:807
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition Frame.cpp:484
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition Frame.cpp:856
Frame()
Constructor - blank frame.
Definition Frame.cpp:59
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition Frame.cpp:700
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition Frame.cpp:847
int GetAudioSamplesCount()
Get number of audio samples.
Definition Frame.cpp:368
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition Frame.cpp:816
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition Frame.cpp:258
int GetHeight()
Get height of image.
Definition Frame.cpp:490
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition Frame.cpp:443
void SetFrameNumber(int64_t number)
Set frame number.
Definition Frame.cpp:450
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition Frame.cpp:374
int64_t number
This is the frame number (starting at 1)
Definition Frame.h:118
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition Frame.cpp:908
This namespace is the default namespace for all code in the openshot library.
Definition Compressor.h:29
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...