26 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0), previous_frame(0)
70 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
73void FrameMapper::AddField(int64_t frame)
76 Field f = { frame, bool(field_toggle) };
80void FrameMapper::AddField(int64_t frame,
bool isOdd)
83 Field f = { frame, isOdd };
87void FrameMapper::AddField(
Field field)
93 field_toggle = (field_toggle ? false :
true);
97void FrameMapper::Clear() {
99 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
112void FrameMapper::Init()
122 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
130 parent_position = parent->
Position();
131 parent_start = parent->
Start();
133 parent_position = 0.0;
145 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
146 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
149 float difference = target.
ToInt() - original.
ToInt();
152 int field_interval = 0;
153 int frame_interval = 0;
157 field_interval = round(fabs(original.
ToInt() / difference));
160 frame_interval = field_interval * 2.0f;
169 for (int64_t field = 1; field <= number_of_fields; field++)
177 else if (difference > 0)
187 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
194 AddField(frame + 1, field_toggle);
196 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
203 else if (difference < 0)
209 field_toggle = (field_toggle ? false :
true);
211 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
216 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
229 if (field % 2 == 0 && field > 0)
242 double original_frame_num = 1.0f;
243 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
246 AddField(round(original_frame_num));
247 AddField(round(original_frame_num));
250 original_frame_num += value_increment;
255 Field Odd = {0,
true};
256 Field Even = {0,
true};
259 int64_t start_samples_frame = 1;
260 int start_samples_position = 0;
262 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
268 if (field % 2 == 0 && field > 0)
271 int64_t frame_number = field / 2;
282 int64_t end_samples_frame = start_samples_frame;
283 int end_samples_position = start_samples_position;
286 while (remaining_samples > 0)
293 if (original_samples >= remaining_samples)
296 end_samples_position += remaining_samples - 1;
297 remaining_samples = 0;
301 end_samples_frame += 1;
302 end_samples_position = 0;
303 remaining_samples -= original_samples;
313 start_samples_frame = end_samples_frame;
314 start_samples_position = end_samples_position + 1;
317 start_samples_frame += 1;
318 start_samples_position = 0;
359 frame.
Odd.
Frame = TargetFrameNumber;
369 if(TargetFrameNumber < 1 ||
frames.size() == 0)
373 else if (TargetFrameNumber > (int64_t)
frames.size())
375 TargetFrameNumber =
frames.size();
379 "FrameMapper::GetMappedFrame",
380 "TargetFrameNumber", TargetFrameNumber,
381 "frames.size()",
frames.size(),
382 "frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
383 "frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
386 return frames[TargetFrameNumber - 1];
390std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
392 std::shared_ptr<Frame> new_frame;
400 "FrameMapper::GetOrCreateFrame (from reader)",
402 "samples_in_frame", samples_in_frame);
405 new_frame = reader->
GetFrame(number);
418 "FrameMapper::GetOrCreateFrame (create blank)",
420 "samples_in_frame", samples_in_frame);
426 new_frame->AddAudioSilence(samples_in_frame);
434 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
435 if (final_frame)
return final_frame;
438 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
442 bool is_increasing =
true;
444 float position = parent->
Position();
445 float start = parent->
Start();
446 if (parent_position != position || parent_start != start) {
462 final_frame = final_cache.
GetFrame(requested_frame);
463 if (final_frame)
return final_frame;
467 int minimum_frames = 1;
471 "FrameMapper::GetFrame (Loop through frames)",
472 "requested_frame", requested_frame,
473 "minimum_frames", minimum_frames);
476 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
480 "FrameMapper::GetFrame (inside omp for loop)",
481 "frame_number", frame_number,
482 "minimum_frames", minimum_frames,
483 "requested_frame", requested_frame);
487 std::shared_ptr<Frame> mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
490 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
491 int samples_in_frame =
Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
500 info.
channels == mapped_frame->GetAudioChannelsCount() &&
502 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
503 mapped.
Samples.
total == samples_in_frame && is_increasing &&
506 mapped_frame->number == frame_number &&
510 final_cache.
Add(mapped_frame);
515 auto frame = std::make_shared<Frame>(
516 frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
517 frame->SampleRate(mapped_frame->SampleRate());
518 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
522 std::shared_ptr<Frame> odd_frame = mapped_frame;
524 if (odd_frame && odd_frame->has_image_data)
525 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()),
true);
528 std::shared_ptr<Frame> even_frame;
529 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
530 if (even_frame && even_frame->has_image_data)
531 frame->AddImage(std::make_shared<QImage>(*even_frame->GetImage()),
false);
535 bool reader_has_audio = frame->SampleRate() > 0 && frame->GetAudioChannelsCount() > 0;
538 bool need_resampling =
false;
544 need_resampling =
true;
552 if (abs(frame->number - previous_frame) > 1) {
563 const int EXTRA_INPUT_SAMPLES = 64;
577 int samples_copied = 0;
582 int remaining_samples = copy_samples.
total - samples_copied;
583 int number_to_copy = 0;
586 std::shared_ptr<Frame> original_frame = mapped_frame;
587 if (starting_frame != original_frame->number) {
588 original_frame = GetOrCreateFrame(starting_frame);
591 int original_samples = original_frame->GetAudioSamplesCount();
594 for (
int channel = 0; channel < channels_in_frame; channel++)
599 number_to_copy = original_samples - copy_samples.
sample_start;
600 if (number_to_copy > remaining_samples)
601 number_to_copy = remaining_samples;
604 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
609 number_to_copy = original_samples;
610 if (number_to_copy > remaining_samples)
611 number_to_copy = remaining_samples;
614 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
620 if (number_to_copy > remaining_samples)
621 number_to_copy = remaining_samples;
624 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
629 samples_copied += number_to_copy;
635 frame->ReverseAudio();
643 final_cache.
Add(frame);
648 return final_cache.
GetFrame(requested_frame);
659 for (
float map = 1; map <=
frames.size(); map++)
662 *out <<
"Target frame #: " << map
663 <<
" mapped to original frame #:\t("
665 << frame.
Even.
Frame <<
" even)" << std::endl;
667 *out <<
" - Audio samples mapped to frame "
702 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
740 root[
"type"] =
"FrameMapper";
759 if (!root[
"reader"].isNull())
765 catch (
const std::exception& e)
768 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
790 "FrameMapper::ChangeMapping",
791 "target_fps.num", target_fps.
num,
792 "target_fps.den", target_fps.
den,
793 "target_pulldown", target_pulldown,
794 "target_sample_rate", target_sample_rate,
795 "target_channels", target_channels,
796 "target_channel_layout", target_channel_layout);
802 target.
num = target_fps.
num;
803 target.
den = target_fps.
den;
809 pulldown = target_pulldown;
840 int total_frame_samples = 0;
841 int channels_in_frame = frame->GetAudioChannelsCount();
842 int sample_rate_in_frame = frame->SampleRate();
843 int samples_in_frame = frame->GetAudioSamplesCount();
844 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
847 "FrameMapper::ResampleMappedAudio",
848 "frame->number", frame->number,
849 "original_frame_number", original_frame_number,
850 "channels_in_frame", channels_in_frame,
851 "samples_in_frame", samples_in_frame,
852 "sample_rate_in_frame", sample_rate_in_frame);
855 float* frame_samples_float = NULL;
857 frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
860 total_frame_samples = samples_in_frame * channels_in_frame;
863 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
868 const int16_t max16 = 32767;
869 const int16_t min16 = -32768;
870 for (
int s = 0; s < total_frame_samples; s++) {
871 valF = frame_samples_float[s] * (1 << 15);
874 else if (valF < min16)
877 conv = int(valF + 32768.5) - 32768;
880 frame_samples[s] = conv;
884 delete[] frame_samples_float;
885 frame_samples_float = NULL;
890 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
892 int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
893 int error_code = avcodec_fill_audio_frame(
894 audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
895 (uint8_t *) frame_samples, buf_size, 1);
900 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) +
"]",
901 "error_code", error_code);
902 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
911 audio_converted->nb_samples = total_frame_samples;
912 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
919 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
921 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
922 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
923 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
925 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
932 audio_converted->data,
933 audio_converted->linesize[0],
934 audio_converted->nb_samples,
936 audio_frame->linesize[0],
937 audio_frame->nb_samples);
940 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
943 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
946 av_freep(&audio_frame->data[0]);
948 av_freep(&audio_converted->data[0]);
950 frame_samples = NULL;
953 int channel_buffer_size = nb_samples;
957 "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
958 "nb_samples", nb_samples,
959 "total_frame_samples", total_frame_samples,
961 "channels_in_frame", channels_in_frame,
966 float *channel_buffer =
new float[channel_buffer_size];
969 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
972 for (
int z = 0; z < channel_buffer_size; z++)
973 channel_buffer[z] = 0.0f;
979 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
982 if (channel_filter == channel)
985 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
1001 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
1009 delete[] channel_buffer;
1010 channel_buffer = NULL;
1013 delete[] resampled_samples;
1014 resampled_samples = NULL;
1017 previous_frame = frame->number;
1021int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1024 float position = 0.0;
1029 start = parent->
Start();
1037 int64_t clip_start_position = round(position *
info.
fps.
ToDouble()) + 1;
1038 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1040 return frame_number;
Header file for Clip class.
Header file for all Exception classes.
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define AV_ALLOCATE_FRAME()
#define AV_RESET_FRAME(av_frame)
Header file for the FrameMapper class.
#define OPEN_MP_NUM_PROCESSORS
Header file for ZeroMQ-based Logger class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Position() const
Get position on timeline (in seconds)
This class represents a clip (used to arrange readers on the timeline)
openshot::Keyframe time
Curve representing the frames over time to play (used for speed and direction of video)
Exception when encoding audio packet.
This class represents a fraction.
int num
Numerator for the fraction.
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
int den
Denominator for the fraction.
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool IsOpen() override
Determine if reader is open or closed.
std::vector< Field > fields
ReaderBase * Reader()
Get the current reader.
std::vector< MappedFrame > frames
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Open() override
Open the internal reader.
void Close() override
Close the openshot::FrameMapper and internal reader.
std::string Json() const override
Generate JSON string of this object.
FrameMapper(ReaderBase *reader, Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Default constructor for openshot::FrameMapper class.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
void SetJson(const std::string value) override
Load JSON string into this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Exception for invalid JSON.
bool IsIncreasing(int index) const
Get the direction of the curve at a specific index (increasing or decreasing)
Exception for frames that are out of bounds.
This abstract class is the base class, used by all readers in libopenshot.
virtual bool IsOpen()=0
Determine if reader is open or closed.
openshot::ReaderInfo info
Information about the current media file.
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
This namespace is the default namespace for all code in the openshot library.
PulldownType
This enumeration determines how frame rates are increased or decreased.
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
This struct holds a single field (half a frame).
This struct holds two fields which together make up a complete video frame.
bool has_single_image
Determines if this file only contains a single image.
float duration
Length of time (in seconds)
int width
The width of the video (in pixesl)
int channels
The number of audio channels used in the audio stream.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
int height
The height of the video (in pixels)
int64_t video_length
The number of frames in the video stream.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
bool has_video
Determines if this file has a video stream.
bool has_audio
Determines if this file has an audio stream.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
This struct holds a the range of samples needed by this frame.
void Extend(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)
Extend SampleRange on either side.
void Shift(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)