31 #include "../include/FrameMapper.h" 37 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
78 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
81 void FrameMapper::AddField(int64_t frame)
84 AddField(
Field(frame, field_toggle));
87 void FrameMapper::AddField(
Field field)
93 field_toggle = (field_toggle ? false :
true);
100 void FrameMapper::Init()
121 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
122 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
125 float difference = target.
ToInt() - original.
ToInt();
128 int field_interval = 0;
129 int frame_interval = 0;
133 field_interval = round(fabs(original.
ToInt() / difference));
136 frame_interval = field_interval * 2.0f;
145 for (int64_t field = 1; field <= number_of_fields; field++)
153 else if (difference > 0)
163 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
170 AddField(
Field(frame + 1, field_toggle));
172 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
179 else if (difference < 0)
185 field_toggle = (field_toggle ? false :
true);
187 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
192 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
205 if (field % 2 == 0 && field > 0)
215 double value_increment = (reader->
info.
video_length + 1) / (
double) (new_length);
218 double original_frame_num = 1.0f;
219 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
222 AddField(round(original_frame_num));
223 AddField(round(original_frame_num));
226 original_frame_num += value_increment;
235 int64_t start_samples_frame = 1;
236 int start_samples_position = 0;
238 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
244 if (field % 2 == 0 && field > 0)
247 int64_t frame_number = field / 2;
258 int64_t end_samples_frame = start_samples_frame;
259 int end_samples_position = start_samples_position;
262 while (remaining_samples > 0)
268 if (original_samples >= remaining_samples)
271 end_samples_position += remaining_samples - 1;
272 remaining_samples = 0;
276 end_samples_frame += 1;
277 end_samples_position = 0;
278 remaining_samples -= original_samples;
288 start_samples_frame = end_samples_frame;
289 start_samples_position = end_samples_position + 1;
292 start_samples_frame += 1;
293 start_samples_position = 0;
326 frame.
Odd.
Frame = TargetFrameNumber;
336 if(TargetFrameNumber < 1 ||
frames.size() == 0)
340 else if (TargetFrameNumber > (int64_t)
frames.size())
342 TargetFrameNumber =
frames.size();
345 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
348 return frames[TargetFrameNumber - 1];
352 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
354 std::shared_ptr<Frame> new_frame;
364 new_frame = reader->
GetFrame(number);
384 new_frame->AddAudioSilence(samples_in_frame);
392 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
393 if (final_frame)
return final_frame;
404 final_frame = final_cache.
GetFrame(requested_frame);
405 if (final_frame)
return final_frame;
409 int minimum_frames = 1;
415 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
419 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame);
423 std::shared_ptr<Frame> mapped_frame;
426 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
429 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
439 info.
channels == mapped_frame->GetAudioChannelsCount() &&
441 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
444 mapped_frame->number == frame_number &&
448 final_cache.
Add(mapped_frame);
453 std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
454 frame->SampleRate(mapped_frame->SampleRate());
455 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
459 std::shared_ptr<Frame> odd_frame;
460 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
463 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
466 std::shared_ptr<Frame> even_frame;
467 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
469 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
473 bool need_resampling =
false;
479 need_resampling =
true;
490 const int EXTRA_INPUT_SAMPLES = 20;
493 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
494 int samples_per_end_frame =
497 if (copy_samples.
sample_end >= samples_per_end_frame)
501 copy_samples.
sample_end -= samples_per_end_frame;
503 copy_samples.
total += EXTRA_INPUT_SAMPLES;
510 int samples_per_start_frame =
513 if (copy_samples.
sample_start >= samples_per_start_frame)
519 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
524 int samples_copied = 0;
529 int remaining_samples = copy_samples.
total - samples_copied;
530 int number_to_copy = 0;
533 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
534 int original_samples = original_frame->GetAudioSamplesCount();
537 for (
int channel = 0; channel < channels_in_frame; channel++)
542 number_to_copy = original_samples - copy_samples.
sample_start;
543 if (number_to_copy > remaining_samples)
544 number_to_copy = remaining_samples;
547 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
552 number_to_copy = original_samples;
553 if (number_to_copy > remaining_samples)
554 number_to_copy = remaining_samples;
557 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
563 if (number_to_copy > remaining_samples)
564 number_to_copy = remaining_samples;
567 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
572 samples_copied += number_to_copy;
582 final_cache.
Add(frame);
587 return final_cache.
GetFrame(requested_frame);
598 float difference = target.
ToInt() - original.
ToInt();
600 int field_interval = 0;
601 int frame_interval = 0;
606 field_interval = round(fabs(original.
ToInt() / difference));
609 frame_interval = field_interval * 2.0f;
613 for (
float map = 1; map <=
frames.size(); map++)
616 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
689 root[
"type"] =
"FrameMapper";
700 Json::CharReaderBuilder rbuilder;
701 Json::CharReader* reader(rbuilder.newCharReader());
704 bool success = reader->parse( value.c_str(),
705 value.c_str() + value.size(), &root, &errors );
710 throw InvalidJSON(
"JSON could not be parsed (or is invalid)");
717 catch (
const std::exception& e)
720 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
741 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
747 target.
num = target_fps.
num;
748 target.
den = target_fps.
den;
753 pulldown = target_pulldown;
781 int total_frame_samples = 0;
782 int channels_in_frame = frame->GetAudioChannelsCount();
783 int sample_rate_in_frame = frame->SampleRate();
784 int samples_in_frame = frame->GetAudioSamplesCount();
785 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
787 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame);
790 float* frame_samples_float = NULL;
792 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
795 total_frame_samples = samples_in_frame * channels_in_frame;
798 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
801 for (
int s = 0; s < total_frame_samples; s++)
803 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
807 delete[] frame_samples_float;
808 frame_samples_float = NULL;
810 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
816 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
818 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
819 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
824 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
830 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
835 audio_converted->nb_samples = total_frame_samples;
836 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
838 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
845 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
847 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
848 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
849 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
851 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
858 audio_converted->data,
859 audio_converted->linesize[0],
860 audio_converted->nb_samples,
862 audio_frame->linesize[0],
863 audio_frame->nb_samples);
866 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
869 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
872 av_freep(&audio_frame->data[0]);
874 av_freep(&audio_converted->data[0]);
876 frame_samples = NULL;
879 int channel_buffer_size = nb_samples;
882 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
885 float *channel_buffer =
new float[channel_buffer_size];
888 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
891 for (
int z = 0; z < channel_buffer_size; z++)
892 channel_buffer[z] = 0.0f;
898 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
901 if (channel_filter == channel)
904 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
920 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
930 delete[] channel_buffer;
931 channel_buffer = NULL;
934 delete[] resampled_samples;
935 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
virtual ~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
std::vector< MappedFrame > frames
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
std::vector< Field > fields
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
void SetJson(std::string value)
Load JSON string into this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
bool IsOpen()
Determine if reader is open or closed.
This class represents a fraction.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
juce::CriticalSection getFrameCriticalSection
Section lock for multiple threads.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
std::string Json()
Get and Set JSON methods.
openshot::ReaderInfo info
Information about the current media file.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
This namespace is the default namespace for all code in the openshot library.
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.