26 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0)
67 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
70 void FrameMapper::AddField(int64_t frame)
73 AddField(
Field(frame, field_toggle));
76 void FrameMapper::AddField(
Field field)
82 field_toggle = (field_toggle ? false :
true);
86 void FrameMapper::Clear() {
88 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
101 void FrameMapper::Init()
111 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
119 parent_position = parent->
Position();
120 parent_start = parent->
Start();
122 parent_position = 0.0;
134 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
135 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
138 float difference = target.
ToInt() - original.
ToInt();
141 int field_interval = 0;
142 int frame_interval = 0;
146 field_interval = round(fabs(original.
ToInt() / difference));
149 frame_interval = field_interval * 2.0f;
158 for (int64_t field = 1; field <= number_of_fields; field++)
166 else if (difference > 0)
176 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
183 AddField(
Field(frame + 1, field_toggle));
185 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
192 else if (difference < 0)
198 field_toggle = (field_toggle ? false :
true);
200 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
205 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
218 if (field % 2 == 0 && field > 0)
231 double original_frame_num = 1.0f;
232 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
235 AddField(round(original_frame_num));
236 AddField(round(original_frame_num));
239 original_frame_num += value_increment;
248 int64_t start_samples_frame = 1;
249 int start_samples_position = 0;
251 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
257 if (field % 2 == 0 && field > 0)
260 int64_t frame_number = field / 2;
271 int64_t end_samples_frame = start_samples_frame;
272 int end_samples_position = start_samples_position;
275 while (remaining_samples > 0)
282 if (original_samples >= remaining_samples)
285 end_samples_position += remaining_samples - 1;
286 remaining_samples = 0;
290 end_samples_frame += 1;
291 end_samples_position = 0;
292 remaining_samples -= original_samples;
302 start_samples_frame = end_samples_frame;
303 start_samples_position = end_samples_position + 1;
306 start_samples_frame += 1;
307 start_samples_position = 0;
341 frame.
Odd.
Frame = TargetFrameNumber;
351 if(TargetFrameNumber < 1 ||
frames.size() == 0)
355 else if (TargetFrameNumber > (int64_t)
frames.size())
357 TargetFrameNumber =
frames.size();
361 "FrameMapper::GetMappedFrame",
362 "TargetFrameNumber", TargetFrameNumber,
363 "frames.size()",
frames.size(),
364 "frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
365 "frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
368 return frames[TargetFrameNumber - 1];
372 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
374 std::shared_ptr<Frame> new_frame;
382 "FrameMapper::GetOrCreateFrame (from reader)",
384 "samples_in_frame", samples_in_frame);
387 new_frame = reader->
GetFrame(number);
400 "FrameMapper::GetOrCreateFrame (create blank)",
402 "samples_in_frame", samples_in_frame);
408 new_frame->AddAudioSilence(samples_in_frame);
416 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
417 if (final_frame)
return final_frame;
420 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
425 float position = parent->
Position();
426 float start = parent->
Start();
427 if (parent_position != position || parent_start != start) {
439 final_frame = final_cache.
GetFrame(requested_frame);
440 if (final_frame)
return final_frame;
444 int minimum_frames = 1;
448 "FrameMapper::GetFrame (Loop through frames)",
449 "requested_frame", requested_frame,
450 "minimum_frames", minimum_frames);
453 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
458 "FrameMapper::GetFrame (inside omp for loop)",
459 "frame_number", frame_number,
460 "minimum_frames", minimum_frames,
461 "requested_frame", requested_frame);
465 std::shared_ptr<Frame> mapped_frame;
468 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
471 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
472 int samples_in_frame =
Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
481 info.
channels == mapped_frame->GetAudioChannelsCount() &&
483 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
486 mapped_frame->number == frame_number &&
490 final_cache.
Add(mapped_frame);
495 auto frame = std::make_shared<Frame>(
496 frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
497 frame->SampleRate(mapped_frame->SampleRate());
498 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
502 std::shared_ptr<Frame> odd_frame;
503 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
506 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()),
true);
509 std::shared_ptr<Frame> even_frame;
510 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
513 std::make_shared<QImage>(*even_frame->GetImage()),
false);
517 bool need_resampling =
false;
523 need_resampling =
true;
534 const int EXTRA_INPUT_SAMPLES = 100;
537 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
538 int samples_per_end_frame =
541 if (copy_samples.
sample_end >= samples_per_end_frame)
545 copy_samples.
sample_end -= samples_per_end_frame;
547 copy_samples.
total += EXTRA_INPUT_SAMPLES;
554 int samples_per_start_frame =
557 if (copy_samples.
sample_start >= samples_per_start_frame)
563 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
568 int samples_copied = 0;
573 int remaining_samples = copy_samples.
total - samples_copied;
574 int number_to_copy = 0;
577 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
578 int original_samples = original_frame->GetAudioSamplesCount();
581 for (
int channel = 0; channel < channels_in_frame; channel++)
586 number_to_copy = original_samples - copy_samples.
sample_start;
587 if (number_to_copy > remaining_samples)
588 number_to_copy = remaining_samples;
591 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
596 number_to_copy = original_samples;
597 if (number_to_copy > remaining_samples)
598 number_to_copy = remaining_samples;
601 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
607 if (number_to_copy > remaining_samples)
608 number_to_copy = remaining_samples;
611 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
616 samples_copied += number_to_copy;
626 final_cache.
Add(frame);
631 return final_cache.
GetFrame(requested_frame);
642 for (
float map = 1; map <=
frames.size(); map++)
645 *out <<
"Target frame #: " << map
646 <<
" mapped to original frame #:\t(" 648 << frame.
Even.
Frame <<
" even)" << std::endl;
650 *out <<
" - Audio samples mapped to frame " 685 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
723 root[
"type"] =
"FrameMapper";
739 catch (
const std::exception& e)
742 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
764 "FrameMapper::ChangeMapping",
765 "target_fps.num", target_fps.
num,
766 "target_fps.den", target_fps.
den,
767 "target_pulldown", target_pulldown,
768 "target_sample_rate", target_sample_rate,
769 "target_channels", target_channels,
770 "target_channel_layout", target_channel_layout);
776 target.
num = target_fps.
num;
777 target.
den = target_fps.
den;
783 pulldown = target_pulldown;
811 int total_frame_samples = 0;
812 int channels_in_frame = frame->GetAudioChannelsCount();
813 int sample_rate_in_frame = frame->SampleRate();
814 int samples_in_frame = frame->GetAudioSamplesCount();
815 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
818 "FrameMapper::ResampleMappedAudio",
819 "frame->number", frame->number,
820 "original_frame_number", original_frame_number,
821 "channels_in_frame", channels_in_frame,
822 "samples_in_frame", samples_in_frame,
823 "sample_rate_in_frame", sample_rate_in_frame);
826 float* frame_samples_float = NULL;
828 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
831 total_frame_samples = samples_in_frame * channels_in_frame;
834 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
839 const int16_t max16 = 32767;
840 const int16_t min16 = -32768;
841 for (
int s = 0; s < total_frame_samples; s++) {
842 valF = frame_samples_float[s] * (1 << 15);
845 else if (valF < min16)
848 conv = int(valF + 32768.5) - 32768;
851 frame_samples[s] = conv;
856 delete[] frame_samples_float;
857 frame_samples_float = NULL;
860 "FrameMapper::ResampleMappedAudio (got sample data from frame)",
861 "frame->number", frame->number,
862 "total_frame_samples", total_frame_samples,
864 "channels_in_frame", channels_in_frame,
866 "samples_in_frame", samples_in_frame);
872 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
874 int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
875 int error_code = avcodec_fill_audio_frame(
876 audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
877 (uint8_t *) frame_samples, buf_size, 1);
882 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) +
"]",
883 "error_code", error_code);
884 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
891 "FrameMapper::ResampleMappedAudio (adjust # of samples)",
892 "total_frame_samples", total_frame_samples,
894 "sample_rate_in_frame", sample_rate_in_frame,
896 "channels_in_frame", channels_in_frame,
897 "original_frame_number", original_frame_number);
902 audio_converted->nb_samples = total_frame_samples;
903 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
906 "FrameMapper::ResampleMappedAudio (preparing for resample)",
907 "in_sample_fmt", AV_SAMPLE_FMT_S16,
908 "out_sample_fmt", AV_SAMPLE_FMT_S16,
909 "in_sample_rate", sample_rate_in_frame,
911 "in_channels", channels_in_frame,
919 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
921 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
922 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
923 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
925 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
932 audio_converted->data,
933 audio_converted->linesize[0],
934 audio_converted->nb_samples,
936 audio_frame->linesize[0],
937 audio_frame->nb_samples);
940 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
943 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
946 av_freep(&audio_frame->data[0]);
948 av_freep(&audio_converted->data[0]);
950 frame_samples = NULL;
953 int channel_buffer_size = nb_samples;
957 "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
958 "nb_samples", nb_samples,
959 "total_frame_samples", total_frame_samples,
961 "channels_in_frame", channels_in_frame,
966 float *channel_buffer =
new float[channel_buffer_size];
969 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
972 for (
int z = 0; z < channel_buffer_size; z++)
973 channel_buffer[z] = 0.0f;
979 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
982 if (channel_filter == channel)
985 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
1001 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
1004 "FrameMapper::ResampleMappedAudio (Add audio to channel)",
1005 "number of samples", position,
1006 "channel_filter", channel_filter);
1014 delete[] channel_buffer;
1015 channel_buffer = NULL;
1018 delete[] resampled_samples;
1019 resampled_samples = NULL;
1023 int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1026 float position = 0.0;
1031 start = parent->
Start();
1039 int64_t clip_start_position = round(position *
info.
fps.
ToDouble()) + 1;
1040 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1042 return frame_number;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
#define AV_FREE_FRAME(av_frame)
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
float Start() const
Get start position (in seconds) of clip (trim start of video)
std::string Json() const override
Generate JSON string of this object.
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
virtual ~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
std::vector< MappedFrame > frames
void Close() override
Close the openshot::FrameMapper and internal reader.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Open() override
Open the internal reader.
std::vector< Field > fields
virtual void Close()=0
Close the reader (and any resources it was consuming)
const Json::Value stringToJson(const std::string value)
This abstract class is the base class, used by all readers in libopenshot.
void SetJson(const std::string value) override
Load JSON string into this object.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
Exception when encoding audio packet.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Header file for all Exception classes.
bool has_audio
Determines if this file has an audio stream.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
This class represents a clip (used to arrange readers on the timeline)
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
Json::Value JsonValue() const override
Generate Json::Value for this object.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Header file for Clip class.
This class represents a fraction.
Header file for the FrameMapper class.
Header file for ZeroMQ-based Logger class.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
This struct holds two fields which together make up a complete video frame.
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
openshot::ReaderInfo info
Information about the current media file.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
This namespace is the default namespace for all code in the openshot library.
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
Exception for invalid JSON.
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
float Position() const
Get position on timeline (in seconds)
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
bool IsOpen() override
Determine if reader is open or closed.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Advanced 2:3:3:2 pull-down (minimal dirty frames)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
virtual bool IsOpen()=0
Determine if reader is open or closed.