24#include <juce_audio_basics/juce_audio_basics.h>
25#include <juce_audio_devices/juce_audio_devices.h>
27#include <QApplication>
45Frame::Frame(int64_t number,
int width,
int height, std::string color,
int samples,
int channels)
46 : audio(std::make_shared<
juce::AudioBuffer<float>>(channels, samples)),
47 number(number), width(width), height(height),
48 pixel_ratio(1,1), color(color),
51 has_audio_data(false), has_image_data(false),
62Frame::Frame(int64_t number,
int width,
int height, std::string color)
63 :
Frame::
Frame(number, width, height, color, 0, 2) {}
67 :
Frame::
Frame(number, 1, 1,
"#000000", samples, channels) {}
90 channels = other.channels;
92 height = other.height;
93 channel_layout = other.channel_layout;
96 sample_rate = other.sample_rate;
97 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
99 max_audio_sample = other.max_audio_sample;
102 image = std::make_shared<QImage>(*(other.image));
104 audio = std::make_shared<juce::AudioBuffer<float>>(*(other.
audio));
105 if (other.wave_image)
106 wave_image = std::make_shared<QImage>(*(other.wave_image));
122 if (!QApplication::instance()) {
125 static char* argv[1] = {NULL};
126 previewApp = std::make_shared<QApplication>(argc, argv);
130 std::shared_ptr<QImage> previewImage =
GetImage();
133 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
136 previewImage = std::make_shared<QImage>(previewImage->scaled(
137 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
138 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
142 QWidget previewWindow;
143 previewWindow.setStyleSheet(
"background-color: #000000;");
148 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150 layout.addWidget(&previewLabel);
153 previewWindow.setLayout(&layout);
154 previewWindow.show();
159std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
165 QVector<QPointF> lines;
166 QVector<QPointF> labels;
170 if (total_samples > 0)
173 int new_height = 200 *
audio->getNumChannels();
174 int height_padding = 20 * (
audio->getNumChannels() - 1);
175 int total_height = new_height + height_padding;
177 float zero_height = 1.0;
181 for (
int channel = 0; channel <
audio->getNumChannels(); channel++)
186 const float *samples =
audio->getReadPointer(channel);
191 float value = samples[sample] * 100.0;
195 if (value > -zero_height && value < 0.0) {
196 value = -zero_height;
197 }
else if (value > 0.0 && value < zero_height) {
202 lines.push_back(QPointF(X, Y));
203 lines.push_back(QPointF(X, Y - value));
207 labels.push_back(QPointF(5.0, Y - 5.0));
210 Y += (200 + height_padding);
215 wave_image = std::make_shared<QImage>(
216 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217 wave_image->fill(QColor(0,0,0,0));
220 QPainter painter(wave_image.get());
224 pen.setColor(QColor(Red, Green, Blue, Alpha));
226 pen.setStyle(Qt::SolidLine);
230 painter.drawLines(lines);
236 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
241 if (wave_image->width() != width || wave_image->height() != height) {
242 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243 wave_image = std::make_shared<QImage>(scaled_wave_image);
261 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
264 return wave_image->constBits();
273 if (!QApplication::instance()) {
276 static char* argv[1] = {NULL};
277 previewApp = std::make_shared<QApplication>(argc, argv);
281 QWidget previewWindow;
282 previewWindow.setStyleSheet(
"background-color: #000000;");
287 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289 layout.addWidget(&previewLabel);
292 previewWindow.setLayout(&layout);
293 previewWindow.show();
305 return audio->getMagnitude(channel, sample, magnitude_range);
309 return audio->getMagnitude(sample, magnitude_range);
320 return buffer->getWritePointer(channel);
329 float *output = NULL;
330 int num_of_channels =
audio->getNumChannels();
334 output =
new float[num_of_channels * num_of_samples];
338 for (
int sample = 0; sample < num_of_samples; sample++)
340 for (
int channel = 0; channel < num_of_channels; channel++)
343 output[position] = buffer->getReadPointer(channel)[sample];
351 *sample_count = num_of_samples;
360 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
362 return audio->getNumChannels();
370 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371 return max_audio_sample;
382 int64_t total_bytes = 0;
384 total_bytes +=
static_cast<int64_t
>(
385 width * height *
sizeof(char) * 4);
389 total_bytes += (sample_rate / 24.0) *
sizeof(
float);
405 return image->constBits();
417 return image->constScanLine(row);
421bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
422 int col_pos = col * 4;
423 if (!image || row < 0 || row >= (height - 1) ||
424 col_pos < 0 || col_pos >= (width - 1) ) {
429 const unsigned char* pixels =
GetPixels(row);
430 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
445 pixel_ratio.
num = num;
446 pixel_ratio.
den = den;
460 if (channels == 0)
return 0;
466 double previous_samples = (sample_rate * fps_rate) * (
number - 1);
467 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
468 previous_samples -= previous_samples_remainder;
471 double total_samples = (sample_rate * fps_rate) *
number;
472 double total_samples_remainder = fmod(total_samples, (
double)channels);
473 total_samples -= total_samples_remainder;
477 int samples_per_frame = round(total_samples - previous_samples);
478 if (samples_per_frame < 0)
479 samples_per_frame = 0;
480 return samples_per_frame;
510 return channel_layout;
518 std::shared_ptr<QImage> previewImage =
GetImage();
521 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
524 previewImage = std::make_shared<QImage>(previewImage->scaled(
525 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
526 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
530 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
533 previewImage = std::make_shared<QImage>(previewImage->scaled(
534 previewImage->size().width() * scale, previewImage->size().height() * scale,
535 Qt::KeepAspectRatio, Qt::SmoothTransformation));
539 previewImage->save(QString::fromStdString(
path), format.c_str(), quality);
543void Frame::Thumbnail(std::string
path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
544 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
547 auto thumbnail = std::make_shared<QImage>(
548 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
549 thumbnail->fill(QColor(QString::fromStdString(background_color)));
552 QPainter painter(thumbnail.get());
553 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
556 std::shared_ptr<QImage> previewImage =
GetImage();
559 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
562 int aspect_width = previewImage->size().width();
563 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
566 previewImage = std::make_shared<QImage>(previewImage->scaled(
567 aspect_width, aspect_height,
568 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
574 previewImage = std::make_shared<QImage>(previewImage->scaled(
575 new_width, new_height,
576 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
579 previewImage = std::make_shared<QImage>(previewImage->scaled(
580 new_width, new_height,
581 Qt::KeepAspectRatio, Qt::SmoothTransformation));
584 int x = (new_width - previewImage->size().width()) / 2.0;
585 int y = (new_height - previewImage->size().height()) / 2.0;
586 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
590 QTransform transform;
591 float origin_x = previewImage->width() / 2.0;
592 float origin_y = previewImage->height() / 2.0;
593 transform.translate(origin_x, origin_y);
594 transform.rotate(rotate);
595 transform.translate(-origin_x,-origin_y);
596 painter.setTransform(transform);
599 painter.drawImage(x, y, *previewImage);
603 if (overlay_path !=
"") {
605 auto overlay = std::make_shared<QImage>();
606 overlay->load(QString::fromStdString(overlay_path));
609 overlay = std::make_shared<QImage>(
610 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
613 overlay = std::make_shared<QImage>(overlay->scaled(
614 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
617 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
618 painter.drawImage(0, 0, *overlay);
623 if (mask_path !=
"") {
625 auto mask = std::make_shared<QImage>();
626 mask->load(QString::fromStdString(mask_path));
629 mask = std::make_shared<QImage>(
630 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
633 mask = std::make_shared<QImage>(mask->scaled(
634 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
637 mask->invertPixels();
640 unsigned char *pixels =
static_cast<unsigned char *
>(thumbnail->bits());
641 const unsigned char *mask_pixels =
static_cast<const unsigned char *
>(mask->constBits());
645 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
648 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
649 int Frame_Alpha = pixels[byte_index + 3];
650 int Mask_Value = constrain(Frame_Alpha - gray_value);
653 pixels[byte_index + 3] = Mask_Value;
662 thumbnail->save(QString::fromStdString(
path), format.c_str(), quality);
666int Frame::constrain(
int color_value)
671 else if (color_value > 255)
679 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
684 AddColor(QColor(QString::fromStdString(new_color)));
691 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
692 image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
695 image->fill(new_color);
701 int new_width,
int new_height,
int bytes_per_pixel,
702 QImage::Format type,
const unsigned char *pixels_)
710 auto new_image = std::make_shared<QImage>(
712 new_width, new_height,
713 new_width * bytes_per_pixel,
715 (QImageCleanupFunction) &openshot::cleanUpBuffer,
729 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
733 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
734 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
737 width = image->width();
738 height = image->height();
757 if (image == new_image || image->size() != new_image->size()) {
760 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
761 new_image = std::make_shared<QImage>(
762 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
769 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
770 unsigned char *pixels = image->bits();
771 const unsigned char *new_pixels = new_image->constBits();
778 for (
int row = start; row < image->height(); row += 2) {
779 int offset = row * image->bytesPerLine();
780 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
784 height = image->height();
785 width = image->width();
794 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
797 audio->setSize(channels, length,
true,
true,
false);
798 channel_layout = layout;
802 max_audio_sample = length;
808 if (
audio && !audio_reversed) {
811 audio_reversed =
true;
816void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
817 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
820 int destStartSampleAdjusted = max(destStartSample, 0);
823 int new_length = destStartSampleAdjusted + numSamples;
824 int new_channel_length =
audio->getNumChannels();
825 if (destChannel >= new_channel_length)
826 new_channel_length = destChannel + 1;
827 if (new_length >
audio->getNumSamples() || new_channel_length >
audio->getNumChannels())
828 audio->setSize(new_channel_length, new_length,
true,
true,
false);
832 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
835 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
839 if (new_length > max_audio_sample)
840 max_audio_sample = new_length;
843 audio_reversed =
false;
847void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
849 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
852 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
871 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
872 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
873 int from_to[] = { 0,0, 1,1, 2,2 };
874 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
875 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
895 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
896 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
898 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
901 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
902 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
922 juce::AudioDeviceManager deviceManager;
923 juce::String error = deviceManager.initialise (
930 if (error.isNotEmpty()) {
931 cout <<
"Error on initialise(): " << error << endl;
934 juce::AudioSourcePlayer audioSourcePlayer;
935 deviceManager.addAudioCallback (&audioSourcePlayer);
937 std::unique_ptr<AudioBufferSource> my_source;
941 juce::TimeSliceThread my_thread(
"Audio buffer thread");
944 my_thread.startThread();
946 juce::AudioTransportSource transport1;
947 transport1.setSource (my_source.get(),
950 (
double) sample_rate,
951 audio->getNumChannels());
952 transport1.setPosition (0);
953 transport1.setGain(1.0);
957 juce::MixerAudioSource mixer;
958 mixer.addInputSource(&transport1,
false);
959 audioSourcePlayer.setSource (&mixer);
964 while (transport1.isPlaying())
966 cout <<
"playing" << endl;
967 std::this_thread::sleep_for(std::chrono::seconds(1));
970 cout <<
"DONE!!!" << endl;
973 transport1.setSource (0);
974 audioSourcePlayer.setSource (0);
975 my_thread.stopThread(500);
976 deviceManager.removeAudioCallback (&audioSourcePlayer);
977 deviceManager.closeAudioDevice();
978 deviceManager.removeAllChangeListeners();
979 deviceManager.dispatchPendingMessages();
981 cout <<
"End of Play()" << endl;
989 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
992 audio->setSize(channels, numSamples,
false,
true,
false);
997 max_audio_sample = numSamples;
1000 audio_reversed =
false;
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
This class represents a fraction.
int num
Numerator for the fraction.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
int den
Denominator for the fraction.
This class represents a single frame of video (i.e. image & audio data)
Frame & operator=(const Frame &other)
Assignment operator.
std::shared_ptr< juce::AudioBuffer< float > > audio
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
const unsigned char * GetPixels()
Get pixel data (as packets)
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
int GetAudioChannelsCount()
Get number of audio channels.
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
bool has_image_data
This frame has been loaded with pixel data.
int GetWidth()
Get height of image.
int SampleRate()
Get the original sample rate of this frame's audio data.
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
void Play()
Play audio samples for this frame.
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
bool has_audio_data
This frame has been loaded with audio data.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
openshot::ChannelLayout ChannelsLayout()
void AddAudioSilence(int numSamples)
Add audio silence.
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void DisplayWaveform()
Display the wave form.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
virtual ~Frame()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Frame()
Constructor - blank frame.
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
int GetAudioSamplesCount()
Get number of audio samples.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
int GetHeight()
Get height of image.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
void SetFrameNumber(int64_t number)
Set frame number.
juce::AudioBuffer< float > * GetAudioSampleBuffer()
int64_t number
This is the frame number (starting at 1)
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
This namespace is the default namespace for all code in the openshot library.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...