OpenShot Library | libopenshot  0.3.0
FrameMapper.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include <cmath>
14 #include <iostream>
15 #include <iomanip>
16 
17 #include "FrameMapper.h"
18 #include "Exceptions.h"
19 #include "Clip.h"
20 #include "ZmqLogger.h"
21 
22 using namespace std;
23 using namespace openshot;
24 
25 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
26  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0)
27 {
28  // Set the original frame rate from the reader
29  original = Fraction(reader->info.fps.num, reader->info.fps.den);
30 
31  // Set all info struct members equal to the internal reader
32  info = reader->info;
33  info.fps.num = target.num;
34  info.fps.den = target.den;
35  info.video_timebase.num = target.den;
36  info.video_timebase.den = target.num;
38  info.sample_rate = target_sample_rate;
39  info.channels = target_channels;
40  info.channel_layout = target_channel_layout;
41  info.width = reader->info.width;
42  info.height = reader->info.height;
43 
44  // Used to toggle odd / even fields
45  field_toggle = true;
46 
47  // Adjust cache size based on size of frame and audio
49 }
50 
51 // Destructor
53 
54  // Auto Close if not already
55  Close();
56 
57  reader = NULL;
58 }
59 
62 {
63  if (reader)
64  return reader;
65  else
66  // Throw error if reader not initialized
67  throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
68 }
69 
70 void FrameMapper::AddField(int64_t frame)
71 {
72  // Add a field, and toggle the odd / even field
73  AddField(Field(frame, field_toggle));
74 }
75 
76 void FrameMapper::AddField(Field field)
77 {
78  // Add a field to the end of the field list
79  fields.push_back(field);
80 
81  // toggle the odd / even flag
82  field_toggle = (field_toggle ? false : true);
83 }
84 
85 // Clear both the fields & frames lists
86 void FrameMapper::Clear() {
87  // Prevent async calls to the following code
88  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
89 
90  // Clear the fields & frames lists
91  fields.clear();
92  fields.shrink_to_fit();
93  frames.clear();
94  frames.shrink_to_fit();
95 }
96 
97 // Use the original and target frame rates and a pull-down technique to create
98 // a mapping between the original fields and frames or a video to a new frame rate.
99 // This might repeat or skip fields and frames of the original video, depending on
100 // whether the frame rate is increasing or decreasing.
101 void FrameMapper::Init()
102 {
103  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
104 
105  // Do not initialize anything if just a picture with no audio
107  // Skip initialization
108  return;
109 
110  // Prevent async calls to the following code
111  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
112 
113  // Clear the fields & frames lists
114  Clear();
115 
116  // Find parent position (if any)
117  Clip *parent = (Clip *) ParentClip();
118  if (parent) {
119  parent_position = parent->Position();
120  parent_start = parent->Start();
121  } else {
122  parent_position = 0.0;
123  parent_start = 0.0;
124  }
125 
126  // Mark as not dirty
127  is_dirty = false;
128 
129  // Clear cache
130  final_cache.Clear();
131 
132  // Some framerates are handled special, and some use a generic Keyframe curve to
133  // map the framerates. These are the special framerates:
134  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
135  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
136 
137  // Get the difference (in frames) between the original and target frame rates
138  float difference = target.ToInt() - original.ToInt();
139 
140  // Find the number (i.e. interval) of fields that need to be skipped or repeated
141  int field_interval = 0;
142  int frame_interval = 0;
143 
144  if (difference != 0)
145  {
146  field_interval = round(fabs(original.ToInt() / difference));
147 
148  // Get frame interval (2 fields per frame)
149  frame_interval = field_interval * 2.0f;
150  }
151 
152 
153  // Calculate # of fields to map
154  int64_t frame = 1;
155  int64_t number_of_fields = reader->info.video_length * 2;
156 
157  // Loop through all fields in the original video file
158  for (int64_t field = 1; field <= number_of_fields; field++)
159  {
160 
161  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
162  {
163  // Add fields
164  AddField(frame);
165  }
166  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
167  {
168  // Add current field
169  AddField(frame);
170 
171  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
172  {
173  // Add extra field for each 'field interval
174  AddField(frame);
175  }
176  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
177  {
178  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
179  AddField(frame); // add field for current frame
180 
181  if (frame + 1 <= info.video_length)
182  // add field for next frame (if the next frame exists)
183  AddField(Field(frame + 1, field_toggle));
184  }
185  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
186  {
187  // No pull-down technique needed, just repeat this frame
188  AddField(frame);
189  AddField(frame);
190  }
191  }
192  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
193  {
194 
195  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
196  {
197  // skip current field and toggle the odd/even flag
198  field_toggle = (field_toggle ? false : true);
199  }
200  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
201  {
202  // skip this field, plus the next field
203  field++;
204  }
205  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
206  {
207  // skip this field, plus the next one
208  field++;
209  }
210  else
211  {
212  // No skipping needed, so add the field
213  AddField(frame);
214  }
215  }
216 
217  // increment frame number (if field is divisible by 2)
218  if (field % 2 == 0 && field > 0)
219  frame++;
220  }
221 
222  } else {
223  // Map the remaining framerates using a linear algorithm
224  double rate_diff = target.ToDouble() / original.ToDouble();
225  int64_t new_length = reader->info.video_length * rate_diff;
226 
227  // Calculate the value difference
228  double value_increment = reader->info.video_length / (double) (new_length);
229 
230  // Loop through curve, and build list of frames
231  double original_frame_num = 1.0f;
232  for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
233  {
234  // Add 2 fields per frame
235  AddField(round(original_frame_num));
236  AddField(round(original_frame_num));
237 
238  // Increment original frame number
239  original_frame_num += value_increment;
240  }
241  }
242 
243  // Loop through the target frames again (combining fields into frames)
244  Field Odd(0, true); // temp field used to track the ODD field
245  Field Even(0, true); // temp field used to track the EVEN field
246 
247  // Variables used to remap audio samples
248  int64_t start_samples_frame = 1;
249  int start_samples_position = 0;
250 
251  for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
252  {
253  // Get the current field
254  Field f = fields[field - 1];
255 
256  // Is field divisible by 2?
257  if (field % 2 == 0 && field > 0)
258  {
259  // New frame number
260  int64_t frame_number = field / 2;
261 
262  // Set the bottom frame
263  if (f.isOdd)
264  Odd = f;
265  else
266  Even = f;
267 
268  // Determine the range of samples (from the original rate). Resampling happens in real-time when
269  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
270  // the original sample rate.
271  int64_t end_samples_frame = start_samples_frame;
272  int end_samples_position = start_samples_position;
273  int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels);
274 
275  while (remaining_samples > 0)
276  {
277  // Get original samples (with NO framerate adjustments)
278  // This is the original reader's frame numbers
279  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
280 
281  // Enough samples
282  if (original_samples >= remaining_samples)
283  {
284  // Take all that we need, and break loop
285  end_samples_position += remaining_samples - 1;
286  remaining_samples = 0;
287  } else
288  {
289  // Not enough samples (take them all, and keep looping)
290  end_samples_frame += 1; // next frame
291  end_samples_position = 0; // next frame, starting on 1st sample
292  remaining_samples -= original_samples; // reduce the remaining amount
293  }
294  }
295 
296 
297 
298  // Create the sample mapping struct
299  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels)};
300 
301  // Reset the audio variables
302  start_samples_frame = end_samples_frame;
303  start_samples_position = end_samples_position + 1;
304  if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame), original, reader->info.sample_rate, reader->info.channels))
305  {
306  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
307  start_samples_position = 0; // reset to 0, since we wrapped
308  }
309 
310  // Create a frame and ADD it to the frames collection
311  MappedFrame frame = {Odd, Even, Samples};
312  frames.push_back(frame);
313  }
314  else
315  {
316  // Set the top field
317  if (f.isOdd)
318  Odd = f;
319  else
320  Even = f;
321  }
322  }
323 
324  // Clear the internal fields list (no longer needed)
325  fields.clear();
326  fields.shrink_to_fit();
327 }
328 
329 MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
330 {
331  // Check if mappings are dirty (and need to be recalculated)
332  if (is_dirty)
333  // Recalculate mappings
334  Init();
335 
336  // Ignore mapping on single image readers
338  // Return the same number
339  MappedFrame frame;
340  frame.Even.Frame = TargetFrameNumber;
341  frame.Odd.Frame = TargetFrameNumber;
342  frame.Samples.frame_start = 0;
343  frame.Samples.frame_end = 0;
344  frame.Samples.sample_start = 0;
345  frame.Samples.sample_end = 0;
346  frame.Samples.total = 0;
347  return frame;
348  }
349 
350  // Check if frame number is valid
351  if(TargetFrameNumber < 1 || frames.size() == 0)
352  // frame too small, return error
353  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
354 
355  else if (TargetFrameNumber > (int64_t)frames.size())
356  // frame too large, set to end frame
357  TargetFrameNumber = frames.size();
358 
359  // Debug output
361  "FrameMapper::GetMappedFrame",
362  "TargetFrameNumber", TargetFrameNumber,
363  "frames.size()", frames.size(),
364  "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame,
365  "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
366 
367  // Return frame
368  return frames[TargetFrameNumber - 1];
369 }
370 
371 // Get or generate a blank frame
372 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
373 {
374  std::shared_ptr<Frame> new_frame;
375 
376  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
377  int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number), target, reader->info.sample_rate, reader->info.channels);
378 
379  try {
380  // Debug output
382  "FrameMapper::GetOrCreateFrame (from reader)",
383  "number", number,
384  "samples_in_frame", samples_in_frame);
385 
386  // Attempt to get a frame (but this could fail if a reader has just been closed)
387  new_frame = reader->GetFrame(number);
388 
389  // Return real frame
390  return new_frame;
391 
392  } catch (const ReaderClosed & e) {
393  // ...
394  } catch (const OutOfBoundsFrame & e) {
395  // ...
396  }
397 
398  // Debug output
400  "FrameMapper::GetOrCreateFrame (create blank)",
401  "number", number,
402  "samples_in_frame", samples_in_frame);
403 
404  // Create blank frame
405  new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
406  new_frame->SampleRate(reader->info.sample_rate);
407  new_frame->ChannelsLayout(info.channel_layout);
408  new_frame->AddAudioSilence(samples_in_frame);
409  return new_frame;
410 }
411 
412 // Get an openshot::Frame object for a specific frame number of this reader.
413 std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
414 {
415  // Check final cache, and just return the frame (if it's available)
416  std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
417  if (final_frame) return final_frame;
418 
419  // Create a scoped lock, allowing only a single thread to run the following code at one time
420  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
421 
422  // Find parent properties (if any)
423  Clip *parent = (Clip *) ParentClip();
424  if (parent) {
425  float position = parent->Position();
426  float start = parent->Start();
427  if (parent_position != position || parent_start != start) {
428  // Force dirty if parent clip has moved or been trimmed
429  // since this heavily affects frame #s and audio mappings
430  is_dirty = true;
431  }
432  }
433 
434  // Check if mappings are dirty (and need to be recalculated)
435  if (is_dirty)
436  Init();
437 
438  // Check final cache a 2nd time (due to potential lock already generating this frame)
439  final_frame = final_cache.GetFrame(requested_frame);
440  if (final_frame) return final_frame;
441 
442  // Minimum number of frames to process (for performance reasons)
443  // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
444  int minimum_frames = 1;
445 
446  // Debug output
448  "FrameMapper::GetFrame (Loop through frames)",
449  "requested_frame", requested_frame,
450  "minimum_frames", minimum_frames);
451 
452  // Loop through all requested frames
453  for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
454  {
455 
456  // Debug output
458  "FrameMapper::GetFrame (inside omp for loop)",
459  "frame_number", frame_number,
460  "minimum_frames", minimum_frames,
461  "requested_frame", requested_frame);
462 
463  // Get the mapped frame
464  MappedFrame mapped = GetMappedFrame(frame_number);
465  std::shared_ptr<Frame> mapped_frame;
466 
467  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
468  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
469 
470  // Get # of channels in the actual frame
471  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
472  int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
473 
474  // Determine if mapped frame is identical to source frame
475  // including audio sample distribution according to mapped.Samples,
476  // and frame_number. In some cases such as end of stream, the reader
477  // will return a frame with a different frame number. In these cases,
478  // we cannot use the frame as is, nor can we modify the frame number,
479  // otherwise the reader's cache object internals become invalid.
480  if (info.sample_rate == mapped_frame->SampleRate() &&
481  info.channels == mapped_frame->GetAudioChannelsCount() &&
482  info.channel_layout == mapped_frame->ChannelsLayout() &&
483  mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
484  mapped.Samples.frame_start == mapped.Odd.Frame &&
485  mapped.Samples.sample_start == 0 &&
486  mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
487  info.fps.num == reader->info.fps.num &&
488  info.fps.den == reader->info.fps.den) {
489  // Add original frame to cache, and skip the rest (for performance reasons)
490  final_cache.Add(mapped_frame);
491  continue;
492  }
493 
494  // Create a new frame
495  auto frame = std::make_shared<Frame>(
496  frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
497  frame->SampleRate(mapped_frame->SampleRate());
498  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
499 
500 
501  // Copy the image from the odd field
502  std::shared_ptr<Frame> odd_frame;
503  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
504 
505  if (odd_frame)
506  frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()), true);
507  if (mapped.Odd.Frame != mapped.Even.Frame) {
508  // Add even lines (if different than the previous image)
509  std::shared_ptr<Frame> even_frame;
510  even_frame = GetOrCreateFrame(mapped.Even.Frame);
511  if (even_frame)
512  frame->AddImage(
513  std::make_shared<QImage>(*even_frame->GetImage()), false);
514  }
515 
516  // Resample audio on frame (if needed)
517  bool need_resampling = false;
518  if (info.has_audio &&
519  (info.sample_rate != frame->SampleRate() ||
520  info.channels != frame->GetAudioChannelsCount() ||
521  info.channel_layout != frame->ChannelsLayout()))
522  // Resample audio and correct # of channels if needed
523  need_resampling = true;
524 
525  // create a copy of mapped.Samples that will be used by copy loop
526  SampleRange copy_samples = mapped.Samples;
527 
528  if (need_resampling)
529  {
530  // Resampling needed, modify copy of SampleRange object that
531  // includes some additional input samples on first iteration,
532  // and continues the offset to ensure that the sample rate
533  // converter isn't input limited.
534  const int EXTRA_INPUT_SAMPLES = 100;
535 
536  // Extend end sample count by an additional EXTRA_INPUT_SAMPLES samples
537  copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
538  int samples_per_end_frame =
539  Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
540  reader->info.sample_rate, reader->info.channels);
541  if (copy_samples.sample_end >= samples_per_end_frame)
542  {
543  // check for wrapping
544  copy_samples.frame_end++;
545  copy_samples.sample_end -= samples_per_end_frame;
546  }
547  copy_samples.total += EXTRA_INPUT_SAMPLES;
548 
549  if (avr) {
550  // Sample rate conversion has been allocated on this clip, so
551  // this is not the first iteration. Extend start position by
552  // EXTRA_INPUT_SAMPLES to keep step with previous frame
553  copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
554  int samples_per_start_frame =
555  Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
556  reader->info.sample_rate, reader->info.channels);
557  if (copy_samples.sample_start >= samples_per_start_frame)
558  {
559  // check for wrapping
560  copy_samples.frame_start++;
561  copy_samples.sample_start -= samples_per_start_frame;
562  }
563  copy_samples.total -= EXTRA_INPUT_SAMPLES;
564  }
565  }
566 
567  // Copy the samples
568  int samples_copied = 0;
569  int64_t starting_frame = copy_samples.frame_start;
570  while (info.has_audio && samples_copied < copy_samples.total)
571  {
572  // Init number of samples to copy this iteration
573  int remaining_samples = copy_samples.total - samples_copied;
574  int number_to_copy = 0;
575 
576  // number of original samples on this frame
577  std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
578  int original_samples = original_frame->GetAudioSamplesCount();
579 
580  // Loop through each channel
581  for (int channel = 0; channel < channels_in_frame; channel++)
582  {
583  if (starting_frame == copy_samples.frame_start)
584  {
585  // Starting frame (take the ending samples)
586  number_to_copy = original_samples - copy_samples.sample_start;
587  if (number_to_copy > remaining_samples)
588  number_to_copy = remaining_samples;
589 
590  // Add samples to new frame
591  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
592  }
593  else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
594  {
595  // Middle frame (take all samples)
596  number_to_copy = original_samples;
597  if (number_to_copy > remaining_samples)
598  number_to_copy = remaining_samples;
599 
600  // Add samples to new frame
601  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
602  }
603  else
604  {
605  // Ending frame (take the beginning samples)
606  number_to_copy = copy_samples.sample_end + 1;
607  if (number_to_copy > remaining_samples)
608  number_to_copy = remaining_samples;
609 
610  // Add samples to new frame
611  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
612  }
613  }
614 
615  // increment frame
616  samples_copied += number_to_copy;
617  starting_frame++;
618  }
619 
620  // Resample audio on frame (if needed)
621  if (need_resampling)
622  // Resample audio and correct # of channels if needed
623  ResampleMappedAudio(frame, mapped.Odd.Frame);
624 
625  // Add frame to final cache
626  final_cache.Add(frame);
627 
628  } // for loop
629 
630  // Return processed openshot::Frame
631  return final_cache.GetFrame(requested_frame);
632 }
633 
634 void FrameMapper::PrintMapping(std::ostream* out)
635 {
636  // Check if mappings are dirty (and need to be recalculated)
637  if (is_dirty)
638  // Recalculate mappings
639  Init();
640 
641  // Loop through frame mappings
642  for (float map = 1; map <= frames.size(); map++)
643  {
644  MappedFrame frame = frames[map - 1];
645  *out << "Target frame #: " << map
646  << " mapped to original frame #:\t("
647  << frame.Odd.Frame << " odd, "
648  << frame.Even.Frame << " even)" << std::endl;
649 
650  *out << " - Audio samples mapped to frame "
651  << frame.Samples.frame_start << ":"
652  << frame.Samples.sample_start << " to frame "
653  << frame.Samples.frame_end << ":"
654  << frame.Samples.sample_end << endl;
655  }
656 
657 }
658 
659 // Determine if reader is open or closed
661  if (reader)
662  return reader->IsOpen();
663  else
664  return false;
665 }
666 
667 // Open the internal reader
669 {
670  if (reader)
671  {
672  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
673 
674  // Open the reader
675  reader->Open();
676  }
677 }
678 
679 // Close the internal reader
681 {
682  if (reader)
683  {
684  // Create a scoped lock, allowing only a single thread to run the following code at one time
685  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
686 
687  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
688 
689  // Close internal reader
690  reader->Close();
691  }
692 
693  // Clear the fields & frames lists
694  Clear();
695 
696  // Mark as dirty
697  is_dirty = true;
698 
699  // Clear cache
700  final_cache.Clear();
701 
702  // Deallocate resample buffer
703  if (avr) {
704  SWR_CLOSE(avr);
705  SWR_FREE(&avr);
706  avr = NULL;
707  }
708 }
709 
710 
711 // Generate JSON string of this object
712 std::string FrameMapper::Json() const {
713 
714  // Return formatted string
715  return JsonValue().toStyledString();
716 }
717 
718 // Generate Json::Value for this object
719 Json::Value FrameMapper::JsonValue() const {
720 
721  // Create root json object
722  Json::Value root = ReaderBase::JsonValue(); // get parent properties
723  root["type"] = "FrameMapper";
724 
725  // return JsonValue
726  return root;
727 }
728 
729 // Load JSON string into this object
730 void FrameMapper::SetJson(const std::string value) {
731 
732  // Parse JSON string into JSON objects
733  try
734  {
735  const Json::Value root = openshot::stringToJson(value);
736  // Set all values that match
737  SetJsonValue(root);
738  }
739  catch (const std::exception& e)
740  {
741  // Error parsing JSON (or missing keys)
742  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
743  }
744 }
745 
746 // Load Json::Value into this object
747 void FrameMapper::SetJsonValue(const Json::Value root) {
748 
749  // Set parent data
751 
752  // Re-Open path, and re-init everything (if needed)
753  if (reader) {
754 
755  Close();
756  Open();
757  }
758 }
759 
760 // Change frame rate or audio mapping details
761 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
762 {
764  "FrameMapper::ChangeMapping",
765  "target_fps.num", target_fps.num,
766  "target_fps.den", target_fps.den,
767  "target_pulldown", target_pulldown,
768  "target_sample_rate", target_sample_rate,
769  "target_channels", target_channels,
770  "target_channel_layout", target_channel_layout);
771 
772  // Mark as dirty
773  is_dirty = true;
774 
775  // Update mapping details
776  target.num = target_fps.num;
777  target.den = target_fps.den;
778  info.fps.num = target_fps.num;
779  info.fps.den = target_fps.den;
780  info.video_timebase.num = target_fps.den;
781  info.video_timebase.den = target_fps.num;
783  pulldown = target_pulldown;
784  info.sample_rate = target_sample_rate;
785  info.channels = target_channels;
786  info.channel_layout = target_channel_layout;
787 
788  // Clear cache
789  final_cache.Clear();
790 
791  // Adjust cache size based on size of frame and audio
793 
794  // Deallocate resample buffer
795  if (avr) {
796  SWR_CLOSE(avr);
797  SWR_FREE(&avr);
798  avr = NULL;
799  }
800 }
801 
802 // Resample audio and map channels (if needed)
803 void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
804 {
805  // Check if mappings are dirty (and need to be recalculated)
806  if (is_dirty)
807  // Recalculate mappings
808  Init();
809 
810  // Init audio buffers / variables
811  int total_frame_samples = 0;
812  int channels_in_frame = frame->GetAudioChannelsCount();
813  int sample_rate_in_frame = frame->SampleRate();
814  int samples_in_frame = frame->GetAudioSamplesCount();
815  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
816 
818  "FrameMapper::ResampleMappedAudio",
819  "frame->number", frame->number,
820  "original_frame_number", original_frame_number,
821  "channels_in_frame", channels_in_frame,
822  "samples_in_frame", samples_in_frame,
823  "sample_rate_in_frame", sample_rate_in_frame);
824 
825  // Get audio sample array
826  float* frame_samples_float = NULL;
827  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
828  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
829 
830  // Calculate total samples
831  total_frame_samples = samples_in_frame * channels_in_frame;
832 
833  // Create a new array (to hold all S16 audio samples for the current queued frames)
834  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
835 
836  // Translate audio sample values back to 16 bit integers with saturation
837  float valF;
838  int16_t conv;
839  const int16_t max16 = 32767;
840  const int16_t min16 = -32768;
841  for (int s = 0; s < total_frame_samples; s++) {
842  valF = frame_samples_float[s] * (1 << 15);
843  if (valF > max16)
844  conv = max16;
845  else if (valF < min16)
846  conv = min16;
847  else
848  conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
849 
850  // Copy into buffer
851  frame_samples[s] = conv;
852  }
853 
854 
855  // Deallocate float array
856  delete[] frame_samples_float;
857  frame_samples_float = NULL;
858 
860  "FrameMapper::ResampleMappedAudio (got sample data from frame)",
861  "frame->number", frame->number,
862  "total_frame_samples", total_frame_samples,
863  "target channels", info.channels,
864  "channels_in_frame", channels_in_frame,
865  "target sample_rate", info.sample_rate,
866  "samples_in_frame", samples_in_frame);
867 
868 
869  // Create input frame (and allocate arrays)
870  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
871  AV_RESET_FRAME(audio_frame);
872  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
873 
874  int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
875  int error_code = avcodec_fill_audio_frame(
876  audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
877  (uint8_t *) frame_samples, buf_size, 1);
878 
879  if (error_code < 0)
880  {
882  "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) + "]",
883  "error_code", error_code);
884  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
885  }
886 
887  // Update total samples & input frame size (due to bigger or smaller data types)
888  total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number), target, info.sample_rate, info.channels);
889 
891  "FrameMapper::ResampleMappedAudio (adjust # of samples)",
892  "total_frame_samples", total_frame_samples,
893  "info.sample_rate", info.sample_rate,
894  "sample_rate_in_frame", sample_rate_in_frame,
895  "info.channels", info.channels,
896  "channels_in_frame", channels_in_frame,
897  "original_frame_number", original_frame_number);
898 
899  // Create output frame (and allocate arrays)
900  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
901  AV_RESET_FRAME(audio_converted);
902  audio_converted->nb_samples = total_frame_samples;
903  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
904 
906  "FrameMapper::ResampleMappedAudio (preparing for resample)",
907  "in_sample_fmt", AV_SAMPLE_FMT_S16,
908  "out_sample_fmt", AV_SAMPLE_FMT_S16,
909  "in_sample_rate", sample_rate_in_frame,
910  "out_sample_rate", info.sample_rate,
911  "in_channels", channels_in_frame,
912  "out_channels", info.channels);
913 
914  int nb_samples = 0;
915 
916  // setup resample context
917  if (!avr) {
918  avr = SWR_ALLOC();
919  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
920  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
921  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
922  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
923  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
924  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
925  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
926  av_opt_set_int(avr, "out_channels", info.channels, 0);
927  SWR_INIT(avr);
928  }
929 
930  // Convert audio samples
931  nb_samples = SWR_CONVERT(avr, // audio resample context
932  audio_converted->data, // output data pointers
933  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
934  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
935  audio_frame->data, // input data pointers
936  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
937  audio_frame->nb_samples); // number of input samples to convert
938 
939  // Create a new array (to hold all resampled S16 audio samples)
940  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
941 
942  // Copy audio samples over original samples
943  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
944 
945  // Free frames
946  av_freep(&audio_frame->data[0]);
947  AV_FREE_FRAME(&audio_frame);
948  av_freep(&audio_converted->data[0]);
949  AV_FREE_FRAME(&audio_converted);
950  frame_samples = NULL;
951 
952  // Resize the frame to hold the right # of channels and samples
953  int channel_buffer_size = nb_samples;
954  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
955 
957  "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
958  "nb_samples", nb_samples,
959  "total_frame_samples", total_frame_samples,
960  "info.sample_rate", info.sample_rate,
961  "channels_in_frame", channels_in_frame,
962  "info.channels", info.channels,
963  "info.channel_layout", info.channel_layout);
964 
965  // Array of floats (to hold samples for each channel)
966  float *channel_buffer = new float[channel_buffer_size];
967 
968  // Divide audio into channels. Loop through each channel
969  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
970  {
971  // Init array
972  for (int z = 0; z < channel_buffer_size; z++)
973  channel_buffer[z] = 0.0f;
974 
975  // Loop through all samples and add them to our Frame based on channel.
976  // Toggle through each channel number, since channel data is stored like (left right left right)
977  int channel = 0;
978  int position = 0;
979  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
980  {
981  // Only add samples for current channel
982  if (channel_filter == channel)
983  {
984  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
985  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
986 
987  // Increment audio position
988  position++;
989  }
990 
991  // increment channel (if needed)
992  if ((channel + 1) < info.channels)
993  // move to next channel
994  channel ++;
995  else
996  // reset channel
997  channel = 0;
998  }
999 
1000  // Add samples to frame for this channel
1001  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
1002 
1004  "FrameMapper::ResampleMappedAudio (Add audio to channel)",
1005  "number of samples", position,
1006  "channel_filter", channel_filter);
1007  }
1008 
1009  // Update frame's audio meta data
1010  frame->SampleRate(info.sample_rate);
1011  frame->ChannelsLayout(info.channel_layout);
1012 
1013  // clear channel buffer
1014  delete[] channel_buffer;
1015  channel_buffer = NULL;
1016 
1017  // Delete arrays
1018  delete[] resampled_samples;
1019  resampled_samples = NULL;
1020 }
1021 
1022 // Adjust frame number for Clip position and start (which can result in a different number)
1023 int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1024 
1025  // Get clip position from parent clip (if any)
1026  float position = 0.0;
1027  float start = 0.0;
1028  Clip *parent = (Clip *) ParentClip();
1029  if (parent) {
1030  position = parent->Position();
1031  start = parent->Start();
1032  }
1033 
1034  // Adjust start frame and position based on parent clip.
1035  // This ensures the same frame # is used by mapped readers and clips,
1036  // when calculating samples per frame.
1037  // Thus, this prevents gaps and mismatches in # of samples.
1038  int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1;
1039  int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1;
1040  int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1041 
1042  return frame_number;
1043 }
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:43
#define AV_FREE_FRAME(av_frame)
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
int num
Numerator for the fraction.
Definition: Fraction.h:32
ReaderBase * Reader()
Get the current reader.
Definition: FrameMapper.cpp:61
#define SWR_INIT(ctx)
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:88
std::string Json() const override
Generate JSON string of this object.
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
virtual ~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:52
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
std::vector< MappedFrame > frames
Definition: FrameMapper.h:155
void Close() override
Close the openshot::FrameMapper and internal reader.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:30
void Open() override
Open the internal reader.
std::vector< Field > fields
Definition: FrameMapper.h:154
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
virtual void Close()=0
Close the reader (and any resources it was consuming)
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:75
void SetJson(const std::string value) override
Load JSON string into this object.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:363
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
This struct holds a single field (half a frame).
Definition: FrameMapper.h:54
Exception when encoding audio packet.
Definition: Exceptions.h:142
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:173
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:74
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:534
Header file for all Exception classes.
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:107
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:90
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
#define AV_ALLOCATE_FRAME()
Json::Value JsonValue() const override
Generate Json::Value for this object.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
Header file for Clip class.
This class represents a fraction.
Definition: Fraction.h:30
Header file for the FrameMapper class.
Header file for ZeroMQ-based Logger class.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:80
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:45
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:162
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:91
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:245
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:88
Exception for frames that are out of bounds.
Definition: Exceptions.h:300
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:35
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
SampleRange Samples
Definition: FrameMapper.h:95
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:45
Exception for invalid JSON.
Definition: Exceptions.h:217
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
#define SWR_ALLOC()
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:46
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:86
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:41
int den
Denominator for the fraction.
Definition: Fraction.h:33
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
bool IsOpen() override
Determine if reader is open or closed.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:44
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
virtual bool IsOpen()=0
Determine if reader is open or closed.