23 #include <AppConfig.h> 24 #include <juce_audio_basics/juce_audio_basics.h> 25 #include <juce_audio_devices/juce_audio_devices.h> 27 #include <QApplication> 35 #include <QHBoxLayout> 45 Frame::Frame(int64_t number,
int width,
int height, std::string color,
int samples,
int channels)
46 : audio(
std::make_shared<
juce::AudioBuffer<float>>(channels, samples)),
47 number(number), width(width), height(height),
48 pixel_ratio(1,1), color(color),
51 has_audio_data(false), has_image_data(false),
63 :
Frame::
Frame(number, width, height, color, 0, 2) {}
67 :
Frame::
Frame(number, 1, 1,
"#000000", samples, channels) {}
90 channels = other.channels;
92 height = other.height;
93 channel_layout = other.channel_layout;
96 sample_rate = other.sample_rate;
97 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
99 max_audio_sample = other.max_audio_sample;
102 image = std::make_shared<QImage>(*(other.image));
104 audio = std::make_shared<juce::AudioBuffer<float>>(*(other.
audio));
105 if (other.wave_image)
106 wave_image = std::make_shared<QImage>(*(other.wave_image));
122 if (!QApplication::instance()) {
125 static char* argv[1] = {NULL};
126 previewApp = std::make_shared<QApplication>(argc, argv);
130 std::shared_ptr<QImage> previewImage =
GetImage();
133 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
136 previewImage = std::make_shared<QImage>(previewImage->scaled(
137 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
138 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
142 QWidget previewWindow;
143 previewWindow.setStyleSheet(
"background-color: #000000;");
148 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150 layout.addWidget(&previewLabel);
153 previewWindow.setLayout(&layout);
154 previewWindow.show();
159 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
165 QVector<QPointF> lines;
166 QVector<QPointF> labels;
170 if (total_samples > 0)
173 int new_height = 200 *
audio->getNumChannels();
174 int height_padding = 20 * (
audio->getNumChannels() - 1);
175 int total_height = new_height + height_padding;
177 float zero_height = 1.0;
181 for (
int channel = 0; channel <
audio->getNumChannels(); channel++)
186 const float *samples =
audio->getReadPointer(channel);
191 float value = samples[sample] * 100.0;
195 if (value > -zero_height && value < 0.0) {
196 value = -zero_height;
197 }
else if (value > 0.0 && value < zero_height) {
202 lines.push_back(QPointF(X, Y));
203 lines.push_back(QPointF(X, Y - value));
207 labels.push_back(QPointF(5.0, Y - 5.0));
210 Y += (200 + height_padding);
215 wave_image = std::make_shared<QImage>(
216 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217 wave_image->fill(QColor(0,0,0,0));
220 QPainter painter(wave_image.get());
224 pen.setColor(QColor(Red, Green, Blue, Alpha));
226 pen.setStyle(Qt::SolidLine);
230 painter.drawLines(lines);
236 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
241 if (wave_image->width() != width || wave_image->height() != height) {
242 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243 wave_image = std::make_shared<QImage>(scaled_wave_image);
261 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
264 return wave_image->constBits();
273 if (!QApplication::instance()) {
276 static char* argv[1] = {NULL};
277 previewApp = std::make_shared<QApplication>(argc, argv);
281 QWidget previewWindow;
282 previewWindow.setStyleSheet(
"background-color: #000000;");
287 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289 layout.addWidget(&previewLabel);
292 previewWindow.setLayout(&layout);
293 previewWindow.show();
305 return audio->getMagnitude(channel, sample, magnitude_range);
309 return audio->getMagnitude(sample, magnitude_range);
317 return audio->getWritePointer(channel);
323 float *output = NULL;
325 int num_of_channels =
audio->getNumChannels();
329 if (new_sample_rate != sample_rate)
338 num_of_samples = buffer->getNumSamples();
342 output =
new float[num_of_channels * num_of_samples];
346 for (
int channel = 0; channel < num_of_channels; channel++)
348 for (
int sample = 0; sample < num_of_samples; sample++)
351 output[position] = buffer->getReadPointer(channel)[sample];
359 *sample_count = num_of_samples;
369 float *output = NULL;
371 int num_of_channels =
audio->getNumChannels();
375 if (new_sample_rate != sample_rate && resampler)
384 num_of_samples = buffer->getNumSamples();
388 output =
new float[num_of_channels * num_of_samples];
392 for (
int sample = 0; sample < num_of_samples; sample++)
394 for (
int channel = 0; channel < num_of_channels; channel++)
397 output[position] = buffer->getReadPointer(channel)[sample];
405 *sample_count = num_of_samples;
414 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
416 return audio->getNumChannels();
424 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
425 return max_audio_sample;
436 int64_t total_bytes = 0;
438 total_bytes +=
static_cast<int64_t
>(
439 width * height *
sizeof(char) * 4);
443 total_bytes += (sample_rate / 24.0) *
sizeof(float);
459 return image->constBits();
471 return image->constScanLine(row);
475 bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
476 int col_pos = col * 4;
477 if (!image || row < 0 || row >= (height - 1) ||
478 col_pos < 0 || col_pos >= (width - 1) ) {
483 const unsigned char* pixels =
GetPixels(row);
484 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
485 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
486 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
487 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
499 pixel_ratio.
num = num;
500 pixel_ratio.
den = den;
516 double previous_samples = (sample_rate * fps_rate) * (number - 1);
517 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
518 previous_samples -= previous_samples_remainder;
521 double total_samples = (sample_rate * fps_rate) * number;
522 double total_samples_remainder = fmod(total_samples, (
double)channels);
523 total_samples -= total_samples_remainder;
527 int samples_per_frame = round(total_samples - previous_samples);
528 if (samples_per_frame < 0)
529 samples_per_frame = 0;
530 return samples_per_frame;
560 return channel_layout;
568 std::shared_ptr<QImage> previewImage =
GetImage();
571 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
574 previewImage = std::make_shared<QImage>(previewImage->scaled(
575 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
576 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
580 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
583 previewImage = std::make_shared<QImage>(previewImage->scaled(
584 previewImage->size().width() * scale, previewImage->size().height() * scale,
585 Qt::KeepAspectRatio, Qt::SmoothTransformation));
589 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
593 void Frame::Thumbnail(std::string
path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
594 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
597 auto thumbnail = std::make_shared<QImage>(
598 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
599 thumbnail->fill(QColor(QString::fromStdString(background_color)));
602 QPainter painter(thumbnail.get());
603 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
606 std::shared_ptr<QImage> previewImage =
GetImage();
609 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
612 int aspect_width = previewImage->size().width();
613 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
616 previewImage = std::make_shared<QImage>(previewImage->scaled(
617 aspect_width, aspect_height,
618 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
624 previewImage = std::make_shared<QImage>(previewImage->scaled(
625 new_width, new_height,
626 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
629 previewImage = std::make_shared<QImage>(previewImage->scaled(
630 new_width, new_height,
631 Qt::KeepAspectRatio, Qt::SmoothTransformation));
634 int x = (new_width - previewImage->size().width()) / 2.0;
635 int y = (new_height - previewImage->size().height()) / 2.0;
636 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
640 QTransform transform;
641 float origin_x = previewImage->width() / 2.0;
642 float origin_y = previewImage->height() / 2.0;
643 transform.translate(origin_x, origin_y);
644 transform.rotate(rotate);
645 transform.translate(-origin_x,-origin_y);
646 painter.setTransform(transform);
649 painter.drawImage(x, y, *previewImage);
653 if (overlay_path !=
"") {
655 auto overlay = std::make_shared<QImage>();
656 overlay->load(QString::fromStdString(overlay_path));
659 overlay = std::make_shared<QImage>(
660 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
663 overlay = std::make_shared<QImage>(overlay->scaled(
664 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
667 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
668 painter.drawImage(0, 0, *overlay);
673 if (mask_path !=
"") {
675 auto mask = std::make_shared<QImage>();
676 mask->load(QString::fromStdString(mask_path));
679 mask = std::make_shared<QImage>(
680 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
683 mask = std::make_shared<QImage>(mask->scaled(
684 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
687 mask->invertPixels();
690 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
691 const unsigned char *mask_pixels = (
const unsigned char *) mask->constBits();
695 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
698 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
699 int Frame_Alpha = pixels[byte_index + 3];
700 int Mask_Value = constrain(Frame_Alpha - gray_value);
703 pixels[byte_index + 3] = Mask_Value;
712 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
716 int Frame::constrain(
int color_value)
721 else if (color_value > 255)
729 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
734 AddColor(QColor(QString::fromStdString(new_color)));
741 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
742 image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
745 image->fill(new_color);
751 int new_width,
int new_height,
int bytes_per_pixel,
752 QImage::Format type,
const unsigned char *pixels_)
760 auto new_image = std::make_shared<QImage>(
762 new_width, new_height,
763 new_width * bytes_per_pixel,
765 (QImageCleanupFunction) &openshot::cleanUpBuffer,
779 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
783 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
784 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
787 width = image->width();
788 height = image->height();
807 if (image == new_image || image->size() != new_image->size()) {
810 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
811 new_image = std::make_shared<QImage>(
812 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
819 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
820 unsigned char *pixels = image->bits();
821 const unsigned char *new_pixels = new_image->constBits();
828 for (
int row = start; row < image->height(); row += 2) {
829 int offset = row * image->bytesPerLine();
830 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
834 height = image->height();
835 width = image->width();
844 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
847 audio->setSize(channels, length,
true,
true,
false);
848 channel_layout = layout;
852 max_audio_sample = length;
856 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
857 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
860 int destStartSampleAdjusted = max(destStartSample, 0);
863 int new_length = destStartSampleAdjusted + numSamples;
864 int new_channel_length =
audio->getNumChannels();
865 if (destChannel >= new_channel_length)
866 new_channel_length = destChannel + 1;
867 if (new_length >
audio->getNumSamples() || new_channel_length >
audio->getNumChannels())
868 audio->setSize(new_channel_length, new_length,
true,
true,
false);
872 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
875 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
879 if (new_length > max_audio_sample)
880 max_audio_sample = new_length;
884 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
886 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
889 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
908 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
909 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
910 int from_to[] = { 0,0, 1,1, 2,2 };
911 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
912 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
932 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
933 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
935 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
938 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
939 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
959 juce::AudioDeviceManager deviceManager;
960 juce::String error = deviceManager.initialise (
967 if (error.isNotEmpty()) {
968 cout <<
"Error on initialise(): " << error << endl;
971 juce::AudioSourcePlayer audioSourcePlayer;
972 deviceManager.addAudioCallback (&audioSourcePlayer);
974 std::unique_ptr<AudioBufferSource> my_source;
978 juce::TimeSliceThread my_thread(
"Audio buffer thread");
981 my_thread.startThread();
983 juce::AudioTransportSource transport1;
984 transport1.setSource (my_source.get(),
987 (double) sample_rate,
988 audio->getNumChannels());
989 transport1.setPosition (0);
990 transport1.setGain(1.0);
994 juce::MixerAudioSource mixer;
995 mixer.addInputSource(&transport1,
false);
996 audioSourcePlayer.setSource (&mixer);
1001 while (transport1.isPlaying())
1003 cout <<
"playing" << endl;
1004 std::this_thread::sleep_for(std::chrono::seconds(1));
1007 cout <<
"DONE!!!" << endl;
1010 transport1.setSource (0);
1011 audioSourcePlayer.setSource (0);
1012 my_thread.stopThread(500);
1013 deviceManager.removeAudioCallback (&audioSourcePlayer);
1014 deviceManager.closeAudioDevice();
1015 deviceManager.removeAllChangeListeners();
1016 deviceManager.dispatchPendingMessages();
1018 cout <<
"End of Play()" << endl;
1026 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
1029 audio->setSize(channels, numSamples,
false,
true,
false);
1034 if (numSamples > max_audio_sample)
1035 max_audio_sample = numSamples;
int GetWidth()
Get height of image.
int num
Numerator for the fraction.
int GetAudioSamplesCount()
Get number of audio samples.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
This class represents a single frame of video (i.e. image & audio data)
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
const unsigned char * GetPixels()
Get pixel data (as packets)
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
void Play()
Play audio samples for this frame.
Header file for AudioBufferSource class.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
int64_t number
This is the frame number (starting at 1)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Header file for Frame class.
void SetBuffer(juce::AudioBuffer< float > *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
float * GetAudioSamples(int channel)
Get an array of sample data.
void SetFrameNumber(int64_t number)
Set frame number.
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Header file for AudioResampler class.
void AddAudioSilence(int numSamples)
Add audio silence.
bool has_audio_data
This frame has been loaded with audio data.
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
This class represents a fraction.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Frame()
Constructor - blank frame.
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
void DisplayWaveform()
Display the wave form.
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Header file for QtUtilities (compatibiity overlay)
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
int GetAudioChannelsCount()
Get number of audio channels.
This namespace is the default namespace for all code in the openshot library.
openshot::ChannelLayout ChannelsLayout()
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
virtual ~Frame()
Destructor.
std::shared_ptr< juce::AudioBuffer< float > > audio
juce::AudioBuffer< float > * GetResampledBuffer()
Get the resampled audio buffer.
int den
Denominator for the fraction.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
juce::AudioBuffer< float > * GetAudioSampleBuffer()
bool has_image_data
This frame has been loaded with pixel data.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
int GetHeight()
Get height of image.
int SampleRate()
Get the original sample rate of this frame's audio data.
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)