OpenShot Library | libopenshot 0.3.3
Loading...
Searching...
No Matches
FrameMapper.cpp
Go to the documentation of this file.
1
9// Copyright (c) 2008-2019 OpenShot Studios, LLC
10//
11// SPDX-License-Identifier: LGPL-3.0-or-later
12
13#include <cmath>
14#include <iostream>
15#include <iomanip>
16
17#include "FrameMapper.h"
18#include "Exceptions.h"
19#include "Clip.h"
20#include "ZmqLogger.h"
21
22using namespace std;
23using namespace openshot;
24
25FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
26 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0), previous_frame(0)
27{
28 // Set the original frame rate from the reader
29 original = Fraction(reader->info.fps.num, reader->info.fps.den);
30
31 // Set all info struct members equal to the internal reader
32 info = reader->info;
33 info.fps.num = target.num;
34 info.fps.den = target.den;
35 info.video_timebase.num = target.den;
36 info.video_timebase.den = target.num;
38 info.sample_rate = target_sample_rate;
39 info.channels = target_channels;
40 info.channel_layout = target_channel_layout;
41 info.width = reader->info.width;
42 info.height = reader->info.height;
43
44 // Enable/Disable audio (based on settings)
46
47 // Used to toggle odd / even fields
48 field_toggle = true;
49
50 // Adjust cache size based on size of frame and audio
52}
53
54// Destructor
56
57 // Auto Close if not already
58 Close();
59
60 reader = NULL;
61}
62
65{
66 if (reader)
67 return reader;
68 else
69 // Throw error if reader not initialized
70 throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
71}
72
73void FrameMapper::AddField(int64_t frame)
74{
75 // Add a field, and toggle the odd / even field
76 Field f = { frame, bool(field_toggle) };
77 AddField(f);
78}
79
80void FrameMapper::AddField(int64_t frame, bool isOdd)
81{
82 // Add a field, and toggle the odd / even field
83 Field f = { frame, isOdd };
84 AddField(f);
85}
86
87void FrameMapper::AddField(Field field)
88{
89 // Add a field to the end of the field list
90 fields.push_back(field);
91
92 // toggle the odd / even flag
93 field_toggle = (field_toggle ? false : true);
94}
95
96// Clear both the fields & frames lists
97void FrameMapper::Clear() {
98 // Prevent async calls to the following code
99 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
100
101 // Clear the fields & frames lists
102 fields.clear();
103 fields.shrink_to_fit();
104 frames.clear();
105 frames.shrink_to_fit();
106}
107
108// Use the original and target frame rates and a pull-down technique to create
109// a mapping between the original fields and frames or a video to a new frame rate.
110// This might repeat or skip fields and frames of the original video, depending on
111// whether the frame rate is increasing or decreasing.
112void FrameMapper::Init()
113{
114 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
115
116 // Do not initialize anything if just a picture with no audio
118 // Skip initialization
119 return;
120
121 // Prevent async calls to the following code
122 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
123
124 // Clear the fields & frames lists
125 Clear();
126
127 // Find parent position (if any)
128 Clip *parent = static_cast<Clip *>(ParentClip());
129 if (parent) {
130 parent_position = parent->Position();
131 parent_start = parent->Start();
132 } else {
133 parent_position = 0.0;
134 parent_start = 0.0;
135 }
136
137 // Mark as not dirty
138 is_dirty = false;
139
140 // Clear cache
141 final_cache.Clear();
142
143 // Some framerates are handled special, and some use a generic Keyframe curve to
144 // map the framerates. These are the special framerates:
145 if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
146 (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
147
148 // Get the difference (in frames) between the original and target frame rates
149 float difference = target.ToInt() - original.ToInt();
150
151 // Find the number (i.e. interval) of fields that need to be skipped or repeated
152 int field_interval = 0;
153 int frame_interval = 0;
154
155 if (difference != 0)
156 {
157 field_interval = round(fabs(original.ToInt() / difference));
158
159 // Get frame interval (2 fields per frame)
160 frame_interval = field_interval * 2.0f;
161 }
162
163
164 // Calculate # of fields to map
165 int64_t frame = 1;
166 int64_t number_of_fields = reader->info.video_length * 2;
167
168 // Loop through all fields in the original video file
169 for (int64_t field = 1; field <= number_of_fields; field++)
170 {
171
172 if (difference == 0) // Same frame rate, NO pull-down or special techniques required
173 {
174 // Add fields
175 AddField(frame);
176 }
177 else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
178 {
179 // Add current field
180 AddField(frame);
181
182 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
183 {
184 // Add extra field for each 'field interval
185 AddField(frame);
186 }
187 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
188 {
189 // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
190 AddField(frame); // add field for current frame
191
192 if (frame + 1 <= info.video_length)
193 // add field for next frame (if the next frame exists)
194 AddField(frame + 1, field_toggle);
195 }
196 else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
197 {
198 // No pull-down technique needed, just repeat this frame
199 AddField(frame);
200 AddField(frame);
201 }
202 }
203 else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
204 {
205
206 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
207 {
208 // skip current field and toggle the odd/even flag
209 field_toggle = (field_toggle ? false : true);
210 }
211 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
212 {
213 // skip this field, plus the next field
214 field++;
215 }
216 else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
217 {
218 // skip this field, plus the next one
219 field++;
220 }
221 else
222 {
223 // No skipping needed, so add the field
224 AddField(frame);
225 }
226 }
227
228 // increment frame number (if field is divisible by 2)
229 if (field % 2 == 0 && field > 0)
230 frame++;
231 }
232
233 } else {
234 // Map the remaining framerates using a linear algorithm
235 double rate_diff = target.ToDouble() / original.ToDouble();
236 int64_t new_length = reader->info.video_length * rate_diff;
237
238 // Calculate the value difference
239 double value_increment = reader->info.video_length / (double) (new_length);
240
241 // Loop through curve, and build list of frames
242 double original_frame_num = 1.0f;
243 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
244 {
245 // Add 2 fields per frame
246 AddField(round(original_frame_num));
247 AddField(round(original_frame_num));
248
249 // Increment original frame number
250 original_frame_num += value_increment;
251 }
252 }
253
254 // Loop through the target frames again (combining fields into frames)
255 Field Odd = {0, true}; // temp field used to track the ODD field
256 Field Even = {0, true}; // temp field used to track the EVEN field
257
258 // Variables used to remap audio samples
259 int64_t start_samples_frame = 1;
260 int start_samples_position = 0;
261
262 for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
263 {
264 // Get the current field
265 Field f = fields[field - 1];
266
267 // Is field divisible by 2?
268 if (field % 2 == 0 && field > 0)
269 {
270 // New frame number
271 int64_t frame_number = field / 2;
272
273 // Set the bottom frame
274 if (f.isOdd)
275 Odd = f;
276 else
277 Even = f;
278
279 // Determine the range of samples (from the original rate). Resampling happens in real-time when
280 // calling the GetFrame() method. So this method only needs to redistribute the original samples with
281 // the original sample rate.
282 int64_t end_samples_frame = start_samples_frame;
283 int end_samples_position = start_samples_position;
284 int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels);
285
286 while (remaining_samples > 0)
287 {
288 // Get original samples (with NO framerate adjustments)
289 // This is the original reader's frame numbers
290 int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
291
292 // Enough samples
293 if (original_samples >= remaining_samples)
294 {
295 // Take all that we need, and break loop
296 end_samples_position += remaining_samples - 1;
297 remaining_samples = 0;
298 } else
299 {
300 // Not enough samples (take them all, and keep looping)
301 end_samples_frame += 1; // next frame
302 end_samples_position = 0; // next frame, starting on 1st sample
303 remaining_samples -= original_samples; // reduce the remaining amount
304 }
305 }
306
307
308
309 // Create the sample mapping struct
310 SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels)};
311
312 // Reset the audio variables
313 start_samples_frame = end_samples_frame;
314 start_samples_position = end_samples_position + 1;
315 if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame), original, reader->info.sample_rate, reader->info.channels))
316 {
317 start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
318 start_samples_position = 0; // reset to 0, since we wrapped
319 }
320
321 // Create a frame and ADD it to the frames collection
322 MappedFrame frame = {Odd, Even, Samples};
323 frames.push_back(frame);
324 }
325 else
326 {
327 // Set the top field
328 if (f.isOdd)
329 Odd = f;
330 else
331 Even = f;
332 }
333 }
334
335 // Clear the internal fields list (no longer needed)
336 fields.clear();
337 fields.shrink_to_fit();
338
339 if (avr) {
340 // Delete resampler (if exists)
341 SWR_CLOSE(avr);
342 SWR_FREE(&avr);
343 avr = NULL;
344 }
345}
346
347MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
348{
349 // Check if mappings are dirty (and need to be recalculated)
350 if (is_dirty)
351 // Recalculate mappings
352 Init();
353
354 // Ignore mapping on single image readers
356 // Return the same number
357 MappedFrame frame;
358 frame.Even.Frame = TargetFrameNumber;
359 frame.Odd.Frame = TargetFrameNumber;
360 frame.Samples.frame_start = 0;
361 frame.Samples.frame_end = 0;
362 frame.Samples.sample_start = 0;
363 frame.Samples.sample_end = 0;
364 frame.Samples.total = 0;
365 return frame;
366 }
367
368 // Check if frame number is valid
369 if(TargetFrameNumber < 1 || frames.size() == 0)
370 // frame too small, return error
371 throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
372
373 else if (TargetFrameNumber > (int64_t)frames.size())
374 // frame too large, set to end frame
375 TargetFrameNumber = frames.size();
376
377 // Debug output
379 "FrameMapper::GetMappedFrame",
380 "TargetFrameNumber", TargetFrameNumber,
381 "frames.size()", frames.size(),
382 "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame,
383 "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
384
385 // Return frame
386 return frames[TargetFrameNumber - 1];
387}
388
389// Get or generate a blank frame
390std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
391{
392 std::shared_ptr<Frame> new_frame;
393
394 // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
395 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number), target, reader->info.sample_rate, reader->info.channels);
396
397 try {
398 // Debug output
400 "FrameMapper::GetOrCreateFrame (from reader)",
401 "number", number,
402 "samples_in_frame", samples_in_frame);
403
404 // Attempt to get a frame (but this could fail if a reader has just been closed)
405 new_frame = reader->GetFrame(number);
406
407 // Return real frame
408 return new_frame;
409
410 } catch (const ReaderClosed & e) {
411 // ...
412 } catch (const OutOfBoundsFrame & e) {
413 // ...
414 }
415
416 // Debug output
418 "FrameMapper::GetOrCreateFrame (create blank)",
419 "number", number,
420 "samples_in_frame", samples_in_frame);
421
422 // Create blank frame
423 new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
424 new_frame->SampleRate(reader->info.sample_rate);
425 new_frame->ChannelsLayout(info.channel_layout);
426 new_frame->AddAudioSilence(samples_in_frame);
427 return new_frame;
428}
429
430// Get an openshot::Frame object for a specific frame number of this reader.
431std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
432{
433 // Check final cache, and just return the frame (if it's available)
434 std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
435 if (final_frame) return final_frame;
436
437 // Create a scoped lock, allowing only a single thread to run the following code at one time
438 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
439
440 // Find parent properties (if any)
441 Clip *parent = static_cast<Clip *>(ParentClip());
442 bool is_increasing = true;
443 if (parent) {
444 float position = parent->Position();
445 float start = parent->Start();
446 if (parent_position != position || parent_start != start) {
447 // Force dirty if parent clip has moved or been trimmed
448 // since this heavily affects frame #s and audio mappings
449 is_dirty = true;
450 }
451
452 // Determine direction of parent clip at this frame (forward or reverse direction)
453 // This is important for reversing audio in our resampler, for smooth reversed audio.
454 is_increasing = parent->time.IsIncreasing(requested_frame);
455 }
456
457 // Check if mappings are dirty (and need to be recalculated)
458 if (is_dirty)
459 Init();
460
461 // Check final cache a 2nd time (due to potential lock already generating this frame)
462 final_frame = final_cache.GetFrame(requested_frame);
463 if (final_frame) return final_frame;
464
465 // Minimum number of frames to process (for performance reasons)
466 // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
467 int minimum_frames = 1;
468
469 // Debug output
471 "FrameMapper::GetFrame (Loop through frames)",
472 "requested_frame", requested_frame,
473 "minimum_frames", minimum_frames);
474
475 // Loop through all requested frames
476 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
477 {
478 // Debug output
480 "FrameMapper::GetFrame (inside omp for loop)",
481 "frame_number", frame_number,
482 "minimum_frames", minimum_frames,
483 "requested_frame", requested_frame);
484
485 // Get the mapped frame
486 MappedFrame mapped = GetMappedFrame(frame_number);
487 std::shared_ptr<Frame> mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
488
489 // Get # of channels in the actual frame
490 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
491 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
492
493 // Determine if mapped frame is identical to source frame
494 // including audio sample distribution according to mapped.Samples,
495 // and frame_number. In some cases such as end of stream, the reader
496 // will return a frame with a different frame number. In these cases,
497 // we cannot use the frame as is, nor can we modify the frame number,
498 // otherwise the reader's cache object internals become invalid.
499 if (info.sample_rate == mapped_frame->SampleRate() &&
500 info.channels == mapped_frame->GetAudioChannelsCount() &&
501 info.channel_layout == mapped_frame->ChannelsLayout() &&
502 mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
503 mapped.Samples.total == samples_in_frame && is_increasing &&
504 mapped.Samples.frame_start == mapped.Odd.Frame &&
505 mapped.Samples.sample_start == 0 &&
506 mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
507 info.fps.num == reader->info.fps.num &&
508 info.fps.den == reader->info.fps.den) {
509 // Add original frame to cache, and skip the rest (for performance reasons)
510 final_cache.Add(mapped_frame);
511 continue;
512 }
513
514 // Create a new frame
515 auto frame = std::make_shared<Frame>(
516 frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
517 frame->SampleRate(mapped_frame->SampleRate());
518 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
519
520
521 // Copy the image from the odd field
522 std::shared_ptr<Frame> odd_frame = mapped_frame;
523
524 if (odd_frame && odd_frame->has_image_data)
525 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()), true);
526 if (mapped.Odd.Frame != mapped.Even.Frame) {
527 // Add even lines (if different than the previous image)
528 std::shared_ptr<Frame> even_frame;
529 even_frame = GetOrCreateFrame(mapped.Even.Frame);
530 if (even_frame && even_frame->has_image_data)
531 frame->AddImage(std::make_shared<QImage>(*even_frame->GetImage()), false);
532 }
533
534 // Determine if reader contains audio samples
535 bool reader_has_audio = frame->SampleRate() > 0 && frame->GetAudioChannelsCount() > 0;
536
537 // Resample audio on frame (if needed)
538 bool need_resampling = false;
539 if ((info.has_audio && reader_has_audio) &&
540 (info.sample_rate != frame->SampleRate() ||
541 info.channels != frame->GetAudioChannelsCount() ||
542 info.channel_layout != frame->ChannelsLayout()))
543 // Resample audio and correct # of channels if needed
544 need_resampling = true;
545
546 // create a copy of mapped.Samples that will be used by copy loop
547 SampleRange copy_samples = mapped.Samples;
548
549 if (need_resampling)
550 {
551 // Check for non-adjacent frame requests - so the resampler can be reset
552 if (abs(frame->number - previous_frame) > 1) {
553 if (avr) {
554 // Delete resampler (if exists)
555 SWR_CLOSE(avr);
556 SWR_FREE(&avr);
557 avr = NULL;
558 }
559 }
560
561 // Resampling needed, modify copy of SampleRange object that includes some additional input samples on
562 // first iteration, and continues the offset to ensure that the resampler is not input limited.
563 const int EXTRA_INPUT_SAMPLES = 64;
564
565 if (!avr) {
566 // This is the first iteration, and we need to extend # of samples for this frame
567 // Extend sample count range by an additional EXTRA_INPUT_SAMPLES
568 copy_samples.Extend(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
569 } else {
570 // Sample rate conversion has already been allocated on this clip, so
571 // this is not the first iteration. Shift position by EXTRA_INPUT_SAMPLES to correctly align samples
572 copy_samples.Shift(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
573 }
574 }
575
576 // Copy the samples
577 int samples_copied = 0;
578 int64_t starting_frame = copy_samples.frame_start;
579 while (info.has_audio && samples_copied < copy_samples.total)
580 {
581 // Init number of samples to copy this iteration
582 int remaining_samples = copy_samples.total - samples_copied;
583 int number_to_copy = 0;
584
585 // number of original samples on this frame
586 std::shared_ptr<Frame> original_frame = mapped_frame;
587 if (starting_frame != original_frame->number) {
588 original_frame = GetOrCreateFrame(starting_frame);
589 }
590
591 int original_samples = original_frame->GetAudioSamplesCount();
592
593 // Loop through each channel
594 for (int channel = 0; channel < channels_in_frame; channel++)
595 {
596 if (starting_frame == copy_samples.frame_start)
597 {
598 // Starting frame (take the ending samples)
599 number_to_copy = original_samples - copy_samples.sample_start;
600 if (number_to_copy > remaining_samples)
601 number_to_copy = remaining_samples;
602
603 // Add samples to new frame
604 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
605 }
606 else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
607 {
608 // Middle frame (take all samples)
609 number_to_copy = original_samples;
610 if (number_to_copy > remaining_samples)
611 number_to_copy = remaining_samples;
612
613 // Add samples to new frame
614 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
615 }
616 else
617 {
618 // Ending frame (take the beginning samples)
619 number_to_copy = copy_samples.sample_end + 1;
620 if (number_to_copy > remaining_samples)
621 number_to_copy = remaining_samples;
622
623 // Add samples to new frame
624 frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
625 }
626 }
627
628 // increment frame
629 samples_copied += number_to_copy;
630 starting_frame++;
631 }
632
633 // Reverse audio (if needed)
634 if (!is_increasing)
635 frame->ReverseAudio();
636
637 // Resample audio on frame (if needed)
638 if (need_resampling)
639 // Resample audio and correct # of channels if needed
640 ResampleMappedAudio(frame, mapped.Odd.Frame);
641
642 // Add frame to final cache
643 final_cache.Add(frame);
644
645 } // for loop
646
647 // Return processed openshot::Frame
648 return final_cache.GetFrame(requested_frame);
649}
650
651void FrameMapper::PrintMapping(std::ostream* out)
652{
653 // Check if mappings are dirty (and need to be recalculated)
654 if (is_dirty)
655 // Recalculate mappings
656 Init();
657
658 // Loop through frame mappings
659 for (float map = 1; map <= frames.size(); map++)
660 {
661 MappedFrame frame = frames[map - 1];
662 *out << "Target frame #: " << map
663 << " mapped to original frame #:\t("
664 << frame.Odd.Frame << " odd, "
665 << frame.Even.Frame << " even)" << std::endl;
666
667 *out << " - Audio samples mapped to frame "
668 << frame.Samples.frame_start << ":"
669 << frame.Samples.sample_start << " to frame "
670 << frame.Samples.frame_end << ":"
671 << frame.Samples.sample_end << endl;
672 }
673
674}
675
676// Determine if reader is open or closed
678 if (reader)
679 return reader->IsOpen();
680 else
681 return false;
682}
683
684// Open the internal reader
686{
687 if (reader)
688 {
689 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
690
691 // Open the reader
692 reader->Open();
693 }
694}
695
696// Close the internal reader
698{
699 if (reader)
700 {
701 // Create a scoped lock, allowing only a single thread to run the following code at one time
702 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
703
704 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
705
706 // Close internal reader
707 reader->Close();
708 }
709
710 // Clear the fields & frames lists
711 Clear();
712
713 // Mark as dirty
714 is_dirty = true;
715
716 // Clear cache
717 final_cache.Clear();
718
719 // Deallocate resample buffer
720 if (avr) {
721 SWR_CLOSE(avr);
722 SWR_FREE(&avr);
723 avr = NULL;
724 }
725}
726
727
728// Generate JSON string of this object
729std::string FrameMapper::Json() const {
730
731 // Return formatted string
732 return JsonValue().toStyledString();
733}
734
735// Generate Json::Value for this object
736Json::Value FrameMapper::JsonValue() const {
737
738 // Create root json object
739 Json::Value root = ReaderBase::JsonValue(); // get parent properties
740 root["type"] = "FrameMapper";
741 if (reader) {
742 root["reader"] = reader->JsonValue();
743 }
744
745 // return JsonValue
746 return root;
747}
748
749// Load JSON string into this object
750void FrameMapper::SetJson(const std::string value) {
751
752 // Parse JSON string into JSON objects
753 try
754 {
755 const Json::Value root = openshot::stringToJson(value);
756 // Set all values that match
757 SetJsonValue(root);
758
759 if (!root["reader"].isNull()) // does Json contain a reader?
760 {
761 // Placeholder to load reader
762 // TODO: need a createReader method for this and Clip JSON methods
763 }
764 }
765 catch (const std::exception& e)
766 {
767 // Error parsing JSON (or missing keys)
768 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
769 }
770}
771
772// Load Json::Value into this object
773void FrameMapper::SetJsonValue(const Json::Value root) {
774
775 // Set parent data
777
778 // Re-Open path, and re-init everything (if needed)
779 if (reader) {
780
781 Close();
782 Open();
783 }
784}
785
786// Change frame rate or audio mapping details
787void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
788{
790 "FrameMapper::ChangeMapping",
791 "target_fps.num", target_fps.num,
792 "target_fps.den", target_fps.den,
793 "target_pulldown", target_pulldown,
794 "target_sample_rate", target_sample_rate,
795 "target_channels", target_channels,
796 "target_channel_layout", target_channel_layout);
797
798 // Mark as dirty
799 is_dirty = true;
800
801 // Update mapping details
802 target.num = target_fps.num;
803 target.den = target_fps.den;
804 info.fps.num = target_fps.num;
805 info.fps.den = target_fps.den;
806 info.video_timebase.num = target_fps.den;
807 info.video_timebase.den = target_fps.num;
809 pulldown = target_pulldown;
810 info.sample_rate = target_sample_rate;
811 info.channels = target_channels;
812 info.channel_layout = target_channel_layout;
813
814 // Enable/Disable audio (based on settings)
816
817 // Clear cache
818 final_cache.Clear();
819
820 // Adjust cache size based on size of frame and audio
822
823 // Deallocate resample buffer
824 if (avr) {
825 SWR_CLOSE(avr);
826 SWR_FREE(&avr);
827 avr = NULL;
828 }
829}
830
831// Resample audio and map channels (if needed)
832void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
833{
834 // Check if mappings are dirty (and need to be recalculated)
835 if (is_dirty)
836 // Recalculate mappings
837 Init();
838
839 // Init audio buffers / variables
840 int total_frame_samples = 0;
841 int channels_in_frame = frame->GetAudioChannelsCount();
842 int sample_rate_in_frame = frame->SampleRate();
843 int samples_in_frame = frame->GetAudioSamplesCount();
844 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
845
847 "FrameMapper::ResampleMappedAudio",
848 "frame->number", frame->number,
849 "original_frame_number", original_frame_number,
850 "channels_in_frame", channels_in_frame,
851 "samples_in_frame", samples_in_frame,
852 "sample_rate_in_frame", sample_rate_in_frame);
853
854 // Get audio sample array
855 float* frame_samples_float = NULL;
856 // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
857 frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
858
859 // Calculate total samples
860 total_frame_samples = samples_in_frame * channels_in_frame;
861
862 // Create a new array (to hold all S16 audio samples for the current queued frames)
863 int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
864
865 // Translate audio sample values back to 16 bit integers with saturation
866 float valF;
867 int16_t conv;
868 const int16_t max16 = 32767;
869 const int16_t min16 = -32768;
870 for (int s = 0; s < total_frame_samples; s++) {
871 valF = frame_samples_float[s] * (1 << 15);
872 if (valF > max16)
873 conv = max16;
874 else if (valF < min16)
875 conv = min16;
876 else
877 conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
878
879 // Copy into buffer
880 frame_samples[s] = conv;
881 }
882
883 // Deallocate float array
884 delete[] frame_samples_float;
885 frame_samples_float = NULL;
886
887 // Create input frame (and allocate arrays)
888 AVFrame *audio_frame = AV_ALLOCATE_FRAME();
889 AV_RESET_FRAME(audio_frame);
890 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
891
892 int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
893 int error_code = avcodec_fill_audio_frame(
894 audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
895 (uint8_t *) frame_samples, buf_size, 1);
896
897 if (error_code < 0)
898 {
900 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) + "]",
901 "error_code", error_code);
902 throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
903 }
904
905 // Update total samples & input frame size (due to bigger or smaller data types)
906 total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number), target, info.sample_rate, info.channels);
907
908 // Create output frame (and allocate arrays)
909 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
910 AV_RESET_FRAME(audio_converted);
911 audio_converted->nb_samples = total_frame_samples;
912 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
913
914 int nb_samples = 0;
915
916 // setup resample context
917 if (!avr) {
918 avr = SWR_ALLOC();
919 av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
920 av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
921 av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
922 av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
923 av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
924 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
925 av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
926 av_opt_set_int(avr, "out_channels", info.channels, 0);
927 SWR_INIT(avr);
928 }
929
930 // Convert audio samples
931 nb_samples = SWR_CONVERT(avr, // audio resample context
932 audio_converted->data, // output data pointers
933 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
934 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
935 audio_frame->data, // input data pointers
936 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
937 audio_frame->nb_samples); // number of input samples to convert
938
939 // Create a new array (to hold all resampled S16 audio samples)
940 int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
941
942 // Copy audio samples over original samples
943 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
944
945 // Free frames
946 av_freep(&audio_frame->data[0]);
947 AV_FREE_FRAME(&audio_frame);
948 av_freep(&audio_converted->data[0]);
949 AV_FREE_FRAME(&audio_converted);
950 frame_samples = NULL;
951
952 // Resize the frame to hold the right # of channels and samples
953 int channel_buffer_size = nb_samples;
954 frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
955
957 "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
958 "nb_samples", nb_samples,
959 "total_frame_samples", total_frame_samples,
960 "info.sample_rate", info.sample_rate,
961 "channels_in_frame", channels_in_frame,
962 "info.channels", info.channels,
963 "info.channel_layout", info.channel_layout);
964
965 // Array of floats (to hold samples for each channel)
966 float *channel_buffer = new float[channel_buffer_size];
967
968 // Divide audio into channels. Loop through each channel
969 for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
970 {
971 // Init array
972 for (int z = 0; z < channel_buffer_size; z++)
973 channel_buffer[z] = 0.0f;
974
975 // Loop through all samples and add them to our Frame based on channel.
976 // Toggle through each channel number, since channel data is stored like (left right left right)
977 int channel = 0;
978 int position = 0;
979 for (int sample = 0; sample < (nb_samples * info.channels); sample++)
980 {
981 // Only add samples for current channel
982 if (channel_filter == channel)
983 {
984 // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
985 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
986
987 // Increment audio position
988 position++;
989 }
990
991 // increment channel (if needed)
992 if ((channel + 1) < info.channels)
993 // move to next channel
994 channel ++;
995 else
996 // reset channel
997 channel = 0;
998 }
999
1000 // Add samples to frame for this channel
1001 frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
1002 }
1003
1004 // Update frame's audio meta data
1005 frame->SampleRate(info.sample_rate);
1006 frame->ChannelsLayout(info.channel_layout);
1007
1008 // clear channel buffer
1009 delete[] channel_buffer;
1010 channel_buffer = NULL;
1011
1012 // Delete arrays
1013 delete[] resampled_samples;
1014 resampled_samples = NULL;
1015
1016 // Keep track of last resampled frame
1017 previous_frame = frame->number;
1018}
1019
1020// Adjust frame number for Clip position and start (which can result in a different number)
1021int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1022
1023 // Get clip position from parent clip (if any)
1024 float position = 0.0;
1025 float start = 0.0;
1026 Clip *parent = static_cast<Clip *>(ParentClip());
1027 if (parent) {
1028 position = parent->Position();
1029 start = parent->Start();
1030 }
1031
1032 // Adjust start frame and position based on parent clip.
1033 // This ensures the same frame # is used by mapped readers and clips,
1034 // when calculating samples per frame.
1035 // Thus, this prevents gaps and mismatches in # of samples.
1036 int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1;
1037 int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1;
1038 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1039
1040 return frame_number;
1041}
Header file for Clip class.
Header file for all Exception classes.
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_ALLOCATE_FRAME()
#define SWR_FREE(ctx)
#define AV_RESET_FRAME(av_frame)
Header file for the FrameMapper class.
#define OPEN_MP_NUM_PROCESSORS
Header file for ZeroMQ-based Logger class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition CacheBase.cpp:30
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition ClipBase.h:88
float Position() const
Get position on timeline (in seconds)
Definition ClipBase.h:86
This class represents a clip (used to arrange readers on the timeline)
Definition Clip.h:89
openshot::Keyframe time
Curve representing the frames over time to play (used for speed and direction of video)
Definition Clip.h:320
Exception when encoding audio packet.
Definition Exceptions.h:143
This class represents a fraction.
Definition Fraction.h:30
int num
Numerator for the fraction.
Definition Fraction.h:32
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition Fraction.cpp:35
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition Fraction.cpp:40
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition Fraction.cpp:45
int den
Denominator for the fraction.
Definition Fraction.h:33
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool IsOpen() override
Determine if reader is open or closed.
std::vector< Field > fields
ReaderBase * Reader()
Get the current reader.
std::vector< MappedFrame > frames
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Open() override
Open the internal reader.
void Close() override
Close the openshot::FrameMapper and internal reader.
std::string Json() const override
Generate JSON string of this object.
FrameMapper(ReaderBase *reader, Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Default constructor for openshot::FrameMapper class.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
void SetJson(const std::string value) override
Load JSON string into this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition Frame.cpp:484
Exception for invalid JSON.
Definition Exceptions.h:218
bool IsIncreasing(int index) const
Get the direction of the curve at a specific index (increasing or decreasing)
Definition KeyFrame.cpp:292
Exception for frames that are out of bounds.
Definition Exceptions.h:301
This abstract class is the base class, used by all readers in libopenshot.
Definition ReaderBase.h:76
virtual bool IsOpen()=0
Determine if reader is open or closed.
openshot::ReaderInfo info
Information about the current media file.
Definition ReaderBase.h:88
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition ReaderBase.h:79
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
Definition Exceptions.h:364
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition ZmqLogger.cpp:35
This namespace is the default namespace for all code in the openshot library.
Definition Compressor.h:29
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition FrameMapper.h:43
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
Definition FrameMapper.h:44
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition FrameMapper.h:45
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition FrameMapper.h:46
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition Json.cpp:16
This struct holds a single field (half a frame).
Definition FrameMapper.h:56
This struct holds two fields which together make up a complete video frame.
bool has_single_image
Determines if this file only contains a single image.
Definition ReaderBase.h:42
float duration
Length of time (in seconds)
Definition ReaderBase.h:43
int width
The width of the video (in pixesl)
Definition ReaderBase.h:46
int channels
The number of audio channels used in the audio stream.
Definition ReaderBase.h:61
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition ReaderBase.h:48
int height
The height of the video (in pixels)
Definition ReaderBase.h:45
int64_t video_length
The number of frames in the video stream.
Definition ReaderBase.h:53
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition ReaderBase.h:62
bool has_video
Determines if this file has a video stream.
Definition ReaderBase.h:40
bool has_audio
Determines if this file has an audio stream.
Definition ReaderBase.h:41
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition ReaderBase.h:55
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition ReaderBase.h:60
This struct holds a the range of samples needed by this frame.
Definition FrameMapper.h:68
void Extend(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)
Extend SampleRange on either side.
Definition FrameMapper.h:76
void Shift(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)