31 #include "../include/Frame.h" 37 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color(
"#000000"),
38 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
42 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
50 : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
55 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
63 number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color(
"#000000"),
68 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
75 Frame::Frame(int64_t
number,
int width,
int height, std::string color,
int samples,
int channels)
76 : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
81 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
108 channels = other.channels;
110 height = other.height;
111 channel_layout = other.channel_layout;
114 sample_rate = other.sample_rate;
115 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
117 max_audio_sample = other.max_audio_sample;
120 image = std::shared_ptr<QImage>(
new QImage(*(other.image)));
122 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(*(other.audio)));
123 if (other.wave_image)
124 wave_image = std::shared_ptr<QImage>(
new QImage(*(other.wave_image)));
137 if (!QApplication::instance()) {
140 static char* argv[1] = {NULL};
141 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
145 std::shared_ptr<QImage> previewImage =
GetImage();
148 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
151 int new_width = previewImage->size().width();
155 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
159 QWidget previewWindow;
160 previewWindow.setStyleSheet(
"background-color: #000000;");
165 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
166 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
167 layout.addWidget(&previewLabel);
170 previewWindow.setLayout(&layout);
171 previewWindow.show();
176 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
182 QVector<QPointF> lines;
183 QVector<QPointF> labels;
187 if (total_samples > 0)
190 int new_height = 200 * audio->getNumChannels();
191 int height_padding = 20 * (audio->getNumChannels() - 1);
192 int total_height = new_height + height_padding;
197 for (
int channel = 0; channel < audio->getNumChannels(); channel++)
202 const float *samples = audio->getReadPointer(channel);
207 float value = samples[sample] * 100;
212 lines.push_back(QPointF(X,Y));
213 lines.push_back(QPointF(X,Y-value));
217 lines.push_back(QPointF(X,Y));
218 lines.push_back(QPointF(X,Y));
223 labels.push_back(QPointF(5, Y - 5));
226 Y += (200 + height_padding);
231 wave_image = std::shared_ptr<QImage>(
new QImage(total_width, total_height, QImage::Format_RGBA8888));
232 wave_image->fill(QColor(0,0,0,0));
235 QPainter painter(wave_image.get());
238 painter.setPen(QColor(Red, Green, Blue, Alpha));
241 painter.drawLines(lines);
254 if (width != total_width || height != total_height) {
255 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
256 wave_image = std::shared_ptr<QImage>(
new QImage(scaled_wave_image));
262 wave_image = std::shared_ptr<QImage>(
new QImage(width, height, QImage::Format_RGBA8888));
263 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
281 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
284 return wave_image->constBits();
293 if (!QApplication::instance()) {
296 static char* argv[1] = {NULL};
297 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
301 QWidget previewWindow;
302 previewWindow.setStyleSheet(
"background-color: #000000;");
307 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
308 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
309 layout.addWidget(&previewLabel);
312 previewWindow.setLayout(&layout);
313 previewWindow.show();
325 return audio->getMagnitude(channel, sample, magnitude_range);
329 return audio->getMagnitude(sample, magnitude_range);
337 return audio->getWritePointer(channel);
343 float *output = NULL;
344 juce::AudioSampleBuffer *buffer(audio.get());
345 int num_of_channels = audio->getNumChannels();
349 if (new_sample_rate != sample_rate)
352 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
358 num_of_samples = buffer->getNumSamples();
362 output =
new float[num_of_channels * num_of_samples];
366 for (
int channel = 0; channel < num_of_channels; channel++)
368 for (
int sample = 0; sample < num_of_samples; sample++)
371 output[position] = buffer->getReadPointer(channel)[sample];
379 *sample_count = num_of_samples;
389 float *output = NULL;
390 juce::AudioSampleBuffer *buffer(audio.get());
391 int num_of_channels = audio->getNumChannels();
395 if (new_sample_rate != sample_rate && resampler)
398 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
404 num_of_samples = buffer->getNumSamples();
408 output =
new float[num_of_channels * num_of_samples];
412 for (
int sample = 0; sample < num_of_samples; sample++)
414 for (
int channel = 0; channel < num_of_channels; channel++)
417 output[position] = buffer->getReadPointer(channel)[sample];
425 *sample_count = num_of_samples;
434 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
436 return audio->getNumChannels();
444 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
445 return max_audio_sample;
456 int64_t total_bytes = 0;
458 total_bytes += (width * height *
sizeof(char) * 4);
461 total_bytes += (sample_rate / 24.0) *
sizeof(float);
477 return image->constBits();
484 return image->constScanLine(row);
488 bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
489 int col_pos = col * 4;
490 if (!image || row < 0 || row >= (height - 1) ||
491 col_pos < 0 || col_pos >= (width - 1) ) {
496 const unsigned char* pixels =
GetPixels(row);
497 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
498 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
499 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
500 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
512 pixel_ratio.
num = num;
513 pixel_ratio.
den = den;
529 double previous_samples = (sample_rate * fps_rate) * (number - 1);
530 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
531 previous_samples -= previous_samples_remainder;
534 double total_samples = (sample_rate * fps_rate) * number;
535 double total_samples_remainder = fmod(total_samples, (
double)channels);
536 total_samples -= total_samples_remainder;
540 int samples_per_frame = round(total_samples - previous_samples);
541 if (samples_per_frame < 0)
542 samples_per_frame = 0;
543 return samples_per_frame;
573 return channel_layout;
581 std::shared_ptr<QImage> previewImage =
GetImage();
584 if (abs(scale) > 1.001 || abs(scale) < 0.999)
586 int new_width = width;
587 int new_height = height;
590 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
593 int new_width = previewImage->size().width();
597 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
601 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
605 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
609 void Frame::Thumbnail(std::string
path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
610 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
613 std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
614 thumbnail->fill(QColor(QString::fromStdString(background_color)));
617 QPainter painter(thumbnail.get());
618 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
621 std::shared_ptr<QImage> previewImage =
GetImage();
624 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
627 int aspect_width = previewImage->size().width();
628 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
631 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
637 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
640 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
643 int x = (new_width - previewImage->size().width()) / 2.0;
644 int y = (new_height - previewImage->size().height()) / 2.0;
645 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
649 QTransform transform;
650 float origin_x = previewImage->width() / 2.0;
651 float origin_y = previewImage->height() / 2.0;
652 transform.translate(origin_x, origin_y);
653 transform.rotate(rotate);
654 transform.translate(-origin_x,-origin_y);
655 painter.setTransform(transform);
658 painter.drawImage(x, y, *previewImage);
662 if (overlay_path !=
"") {
664 std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(
new QImage());
665 overlay->load(QString::fromStdString(overlay_path));
668 overlay = std::shared_ptr<QImage>(
new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
671 overlay = std::shared_ptr<QImage>(
new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
674 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
675 painter.drawImage(0, 0, *overlay);
680 if (mask_path !=
"") {
682 std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(
new QImage());
683 mask->load(QString::fromStdString(mask_path));
686 mask = std::shared_ptr<QImage>(
new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
689 mask = std::shared_ptr<QImage>(
new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
692 mask->invertPixels();
695 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
696 const unsigned char *mask_pixels = (
const unsigned char *) mask->constBits();
700 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
703 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
704 int Frame_Alpha = pixels[byte_index + 3];
705 int Mask_Value = constrain(Frame_Alpha - gray_value);
708 pixels[byte_index + 3] = Mask_Value;
717 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
721 int Frame::constrain(
int color_value)
726 else if (color_value > 255)
739 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
740 #pragma omp critical (AddImage) 742 image = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
745 image->fill(QColor(QString::fromStdString(color)));
748 width = image->width();
749 height = image->height();
754 void Frame::AddImage(
int new_width,
int new_height,
int bytes_per_pixel, QImage::Format type,
const unsigned char *pixels_)
757 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
758 int buffer_size = new_width * new_height * bytes_per_pixel;
759 qbuffer =
new unsigned char[buffer_size]();
762 memcpy((
unsigned char*)qbuffer, pixels_, buffer_size);
765 #pragma omp critical (AddImage) 767 image = std::shared_ptr<QImage>(
new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &
openshot::Frame::cleanUpBuffer, (
void*) qbuffer));
770 if (image->format() != QImage::Format_RGBA8888)
771 *image = image->convertToFormat(QImage::Format_RGBA8888);
774 width = image->width();
775 height = image->height();
788 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
789 #pragma omp critical (AddImage) 794 if (image->format() != QImage::Format_RGBA8888)
795 *image = image->convertToFormat(QImage::Format_RGBA8888);
798 width = image->width();
799 height = image->height();
820 #pragma omp critical (AddImage) 821 if (image == new_image || image->size() != image->size() || image->format() != image->format())
827 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
828 #pragma omp critical (AddImage) 830 const unsigned char *pixels = image->constBits();
831 const unsigned char *new_pixels = new_image->constBits();
838 for (
int row = start; row < image->height(); row += 2) {
839 memcpy((
unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
840 new_pixels += image->bytesPerLine();
844 width = image->width();
845 height = image->height();
855 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
858 audio->setSize(channels, length,
true,
true,
false);
859 channel_layout = layout;
863 max_audio_sample = length;
867 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
868 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
869 #pragma omp critical (adding_audio) 872 int destStartSampleAdjusted = max(destStartSample, 0);
875 int new_length = destStartSampleAdjusted + numSamples;
876 int new_channel_length = audio->getNumChannels();
877 if (destChannel >= new_channel_length)
878 new_channel_length = destChannel + 1;
879 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
880 audio->setSize(new_channel_length, new_length,
true,
true,
false);
884 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
887 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
891 if (new_length > max_audio_sample)
892 max_audio_sample = new_length;
897 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
899 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
902 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
916 #ifdef USE_IMAGEMAGICK 926 const QRgb *tmpBits = (
const QRgb*)image->constBits();
929 std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(
new Magick::Image(image->width(), image->height(),
"RGBA", Magick::CharPixel, tmpBits));
932 magick_image->backgroundColor(Magick::Color(
"none"));
933 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
940 #ifdef USE_IMAGEMAGICK 945 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
950 qbuffer =
new unsigned char[bufferSize]();
951 unsigned char *buffer = (
unsigned char*)qbuffer;
953 MagickCore::ExceptionInfo exception;
955 MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(),
"RGBA", Magick::CharPixel, buffer, &exception);
958 image = std::shared_ptr<QImage>(
new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &
cleanUpBuffer, (
void*) qbuffer));
961 width = image->width();
962 height = image->height();
974 juce::AudioDeviceManager deviceManager;
975 String error = deviceManager.initialise (0,
981 if (error.isNotEmpty()) {
982 cout <<
"Error on initialise(): " << error.toStdString() << endl;
985 juce::AudioSourcePlayer audioSourcePlayer;
986 deviceManager.addAudioCallback (&audioSourcePlayer);
988 ScopedPointer<AudioBufferSource> my_source;
992 juce::TimeSliceThread my_thread(
"Audio buffer thread");
995 my_thread.startThread();
997 AudioTransportSource transport1;
998 transport1.setSource (my_source,
1001 (
double) sample_rate,
1002 audio->getNumChannels());
1003 transport1.setPosition (0);
1004 transport1.setGain(1.0);
1008 juce::MixerAudioSource mixer;
1009 mixer.addInputSource(&transport1,
false);
1010 audioSourcePlayer.setSource (&mixer);
1015 while (transport1.isPlaying())
1017 cout <<
"playing" << endl;
1021 cout <<
"DONE!!!" << endl;
1024 transport1.setSource (0);
1025 audioSourcePlayer.setSource (0);
1026 my_thread.stopThread(500);
1027 deviceManager.removeAudioCallback (&audioSourcePlayer);
1028 deviceManager.closeAudioDevice();
1029 deviceManager.removeAllChangeListeners();
1030 deviceManager.dispatchPendingMessages();
1032 cout <<
"End of Play()" << endl;
1043 unsigned char* ptr_to_qbuffer = (
unsigned char*) info;
1044 delete[] ptr_to_qbuffer;
1051 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1054 audio->setSize(channels, numSamples,
false,
true,
false);
1059 if (numSamples > max_audio_sample)
1060 max_audio_sample = numSamples;
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
int GetWidth()
Get height of image.
int num
Numerator for the fraction.
int GetAudioSamplesCount()
Get number of audio samples.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
#define MAGICK_IMAGE_ALPHA(im, a)
This class represents a single frame of video (i.e. image & audio data)
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
juce::AudioSampleBuffer * GetAudioSampleBuffer()
const unsigned char * GetPixels()
Get pixel data (as packets)
void Play()
Play audio samples for this frame.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Fraction Reciprocal()
Return the reciprocal as a Fraction.
int64_t number
This is the frame number (starting at 1)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
float * GetAudioSamples(int channel)
Get an array of sample data.
void SetFrameNumber(int64_t number)
Set frame number.
void AddAudioSilence(int numSamples)
Add audio silence.
bool has_audio_data
This frame has been loaded with audio data.
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
This class represents a fraction.
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
void DisplayWaveform()
Display the wave form.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
int GetAudioChannelsCount()
Get number of audio channels.
This namespace is the default namespace for all code in the openshot library.
openshot::ChannelLayout ChannelsLayout()
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
virtual ~Frame()
Destructor.
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
int den
Denominator for the fraction.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
bool has_image_data
This frame has been loaded with pixel data.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
int GetHeight()
Get height of image.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int SampleRate()
Get the original sample rate of this frame's audio data.
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)