OpenShot Library | libopenshot  0.3.0
Frame.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include <thread> // for std::this_thread::sleep_for
14 #include <chrono> // for std::chrono::milliseconds
15 #include <sstream>
16 #include <iomanip>
17 
18 #include "Frame.h"
19 #include "AudioBufferSource.h"
20 #include "AudioResampler.h"
21 #include "QtUtilities.h"
22 
23 #include <AppConfig.h>
24 #include <juce_audio_basics/juce_audio_basics.h>
25 #include <juce_audio_devices/juce_audio_devices.h>
26 
27 #include <QApplication>
28 #include <QImage>
29 #include <QPixmap>
30 #include <QBitmap>
31 #include <QColor>
32 #include <QString>
33 #include <QVector>
34 #include <QPainter>
35 #include <QHBoxLayout>
36 #include <QWidget>
37 #include <QLabel>
38 #include <QPointF>
39 #include <QWidget>
40 
41 using namespace std;
42 using namespace openshot;
43 
44 // Constructor - image & audio
45 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
46  : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
47  number(number), width(width), height(height),
48  pixel_ratio(1,1), color(color),
49  channels(channels), channel_layout(LAYOUT_STEREO),
50  sample_rate(44100),
51  has_audio_data(false), has_image_data(false),
52  max_audio_sample(0)
53 {
54  // zero (fill with silence) the audio buffer
55  audio->clear();
56 }
57 
58 // Delegating Constructor - blank frame
59 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
60 
61 // Delegating Constructor - image only
62 Frame::Frame(int64_t number, int width, int height, std::string color)
63  : Frame::Frame(number, width, height, color, 0, 2) {}
64 
65 // Delegating Constructor - audio only
66 Frame::Frame(int64_t number, int samples, int channels)
67  : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
68 
69 
70 // Copy constructor
71 Frame::Frame ( const Frame &other )
72 {
73  // copy pointers and data
74  DeepCopy(other);
75 }
76 
77 // Assignment operator
79 {
80  // copy pointers and data
81  DeepCopy(other);
82 
83  return *this;
84 }
85 
86 // Copy data and pointers from another Frame instance
87 void Frame::DeepCopy(const Frame& other)
88 {
89  number = other.number;
90  channels = other.channels;
91  width = other.width;
92  height = other.height;
93  channel_layout = other.channel_layout;
96  sample_rate = other.sample_rate;
97  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
98  color = other.color;
99  max_audio_sample = other.max_audio_sample;
100 
101  if (other.image)
102  image = std::make_shared<QImage>(*(other.image));
103  if (other.audio)
104  audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
105  if (other.wave_image)
106  wave_image = std::make_shared<QImage>(*(other.wave_image));
107 }
108 
109 // Destructor
111  // Clear all pointers
112  image.reset();
113  audio.reset();
114  #ifdef USE_OPENCV
115  imagecv.release();
116  #endif
117 }
118 
119 // Display the frame image to the screen (primarily used for debugging reasons)
121 {
122  if (!QApplication::instance()) {
123  // Only create the QApplication once
124  static int argc = 1;
125  static char* argv[1] = {NULL};
126  previewApp = std::make_shared<QApplication>(argc, argv);
127  }
128 
129  // Get preview image
130  std::shared_ptr<QImage> previewImage = GetImage();
131 
132  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
133  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
134  {
135  // Resize to fix DAR
136  previewImage = std::make_shared<QImage>(previewImage->scaled(
137  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
138  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
139  }
140 
141  // Create window
142  QWidget previewWindow;
143  previewWindow.setStyleSheet("background-color: #000000;");
144  QHBoxLayout layout;
145 
146  // Create label with current frame's image
147  QLabel previewLabel;
148  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150  layout.addWidget(&previewLabel);
151 
152  // Show the window
153  previewWindow.setLayout(&layout);
154  previewWindow.show();
155  previewApp->exec();
156 }
157 
158 // Get an audio waveform image
159 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
160 {
161  // Clear any existing waveform image
162  ClearWaveform();
163 
164  // Init a list of lines
165  QVector<QPointF> lines;
166  QVector<QPointF> labels;
167 
168  // Calculate width of an image based on the # of samples
169  int total_samples = GetAudioSamplesCount();
170  if (total_samples > 0)
171  {
172  // If samples are present...
173  int new_height = 200 * audio->getNumChannels();
174  int height_padding = 20 * (audio->getNumChannels() - 1);
175  int total_height = new_height + height_padding;
176  int total_width = 0;
177  float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
178 
179  // Loop through each audio channel
180  float Y = 100.0;
181  for (int channel = 0; channel < audio->getNumChannels(); channel++)
182  {
183  float X = 0.0;
184 
185  // Get audio for this channel
186  const float *samples = audio->getReadPointer(channel);
187 
188  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
189  {
190  // Sample value (scaled to -100 to 100)
191  float value = samples[sample] * 100.0;
192 
193  // Set threshold near zero (so we don't allow near-zero values)
194  // This prevents empty gaps from appearing in the waveform
195  if (value > -zero_height && value < 0.0) {
196  value = -zero_height;
197  } else if (value > 0.0 && value < zero_height) {
198  value = zero_height;
199  }
200 
201  // Append a line segment for each sample
202  lines.push_back(QPointF(X, Y));
203  lines.push_back(QPointF(X, Y - value));
204  }
205 
206  // Add Channel Label Coordinate
207  labels.push_back(QPointF(5.0, Y - 5.0));
208 
209  // Increment Y
210  Y += (200 + height_padding);
211  total_width = X;
212  }
213 
214  // Create blank image
215  wave_image = std::make_shared<QImage>(
216  total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217  wave_image->fill(QColor(0,0,0,0));
218 
219  // Load QPainter with wave_image device
220  QPainter painter(wave_image.get());
221 
222  // Set pen color
223  QPen pen;
224  pen.setColor(QColor(Red, Green, Blue, Alpha));
225  pen.setWidthF(1.0);
226  pen.setStyle(Qt::SolidLine);
227  painter.setPen(pen);
228 
229  // Draw the waveform
230  painter.drawLines(lines);
231  painter.end();
232  }
233  else
234  {
235  // No audio samples present
236  wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237  wave_image->fill(QColor(QString::fromStdString("#000000")));
238  }
239 
240  // Resize Image (if needed)
241  if (wave_image->width() != width || wave_image->height() != height) {
242  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243  wave_image = std::make_shared<QImage>(scaled_wave_image);
244  }
245 
246  // Return new image
247  return wave_image;
248 }
249 
250 // Clear the waveform image (and deallocate its memory)
252 {
253  if (wave_image)
254  wave_image.reset();
255 }
256 
257 // Get an audio waveform image pixels
258 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
259 {
260  // Get audio wave form image
261  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
262 
263  // Return array of pixel packets
264  return wave_image->constBits();
265 }
266 
267 // Display the wave form
269 {
270  // Get audio wave form image
271  GetWaveform(720, 480, 0, 123, 255, 255);
272 
273  if (!QApplication::instance()) {
274  // Only create the QApplication once
275  static int argc = 1;
276  static char* argv[1] = {NULL};
277  previewApp = std::make_shared<QApplication>(argc, argv);
278  }
279 
280  // Create window
281  QWidget previewWindow;
282  previewWindow.setStyleSheet("background-color: #000000;");
283  QHBoxLayout layout;
284 
285  // Create label with current frame's waveform image
286  QLabel previewLabel;
287  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289  layout.addWidget(&previewLabel);
290 
291  // Show the window
292  previewWindow.setLayout(&layout);
293  previewWindow.show();
294  previewApp->exec();
295 
296  // Deallocate waveform image
297  ClearWaveform();
298 }
299 
300 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
301 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
302 {
303  if (channel > 0) {
304  // return average magnitude for a specific channel/sample range
305  return audio->getMagnitude(channel, sample, magnitude_range);
306 
307  } else {
308  // Return average magnitude for all channels
309  return audio->getMagnitude(sample, magnitude_range);
310  }
311 }
312 
313 // Get an array of sample data
314 float* Frame::GetAudioSamples(int channel)
315 {
316  // return JUCE audio data for this channel
317  return audio->getWritePointer(channel);
318 }
319 
320 // Get a planar array of sample data, using any sample rate
321 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
322 {
323  float *output = NULL;
324  juce::AudioBuffer<float> *buffer(audio.get());
325  int num_of_channels = audio->getNumChannels();
326  int num_of_samples = GetAudioSamplesCount();
327 
328  // Resample to new sample rate (if needed)
329  if (new_sample_rate != sample_rate)
330  {
331  // YES, RESAMPLE AUDIO
332  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
333 
334  // Resample data, and return new buffer pointer
335  buffer = resampler->GetResampledBuffer();
336 
337  // Update num_of_samples
338  num_of_samples = buffer->getNumSamples();
339  }
340 
341  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
342  output = new float[num_of_channels * num_of_samples];
343  int position = 0;
344 
345  // Loop through samples in each channel (combining them)
346  for (int channel = 0; channel < num_of_channels; channel++)
347  {
348  for (int sample = 0; sample < num_of_samples; sample++)
349  {
350  // Add sample to output array
351  output[position] = buffer->getReadPointer(channel)[sample];
352 
353  // increment position
354  position++;
355  }
356  }
357 
358  // Update sample count (since it might have changed due to resampling)
359  *sample_count = num_of_samples;
360 
361  // return combined array
362  return output;
363 }
364 
365 
366 // Get an array of sample data (all channels interleaved together), using any sample rate
367 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
368 {
369  float *output = NULL;
370  juce::AudioBuffer<float> *buffer(audio.get());
371  int num_of_channels = audio->getNumChannels();
372  int num_of_samples = GetAudioSamplesCount();
373 
374  // Resample to new sample rate (if needed)
375  if (new_sample_rate != sample_rate && resampler)
376  {
377  // YES, RESAMPLE AUDIO
378  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
379 
380  // Resample data, and return new buffer pointer
381  buffer = resampler->GetResampledBuffer();
382 
383  // Update num_of_samples
384  num_of_samples = buffer->getNumSamples();
385  }
386 
387  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
388  output = new float[num_of_channels * num_of_samples];
389  int position = 0;
390 
391  // Loop through samples in each channel (combining them)
392  for (int sample = 0; sample < num_of_samples; sample++)
393  {
394  for (int channel = 0; channel < num_of_channels; channel++)
395  {
396  // Add sample to output array
397  output[position] = buffer->getReadPointer(channel)[sample];
398 
399  // increment position
400  position++;
401  }
402  }
403 
404  // Update sample count (since it might have changed due to resampling)
405  *sample_count = num_of_samples;
406 
407  // return combined array
408  return output;
409 }
410 
411 // Get number of audio channels
413 {
414  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
415  if (audio)
416  return audio->getNumChannels();
417  else
418  return 0;
419 }
420 
421 // Get number of audio samples
423 {
424  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
425  return max_audio_sample;
426 }
427 
429 {
430  return audio.get();
431 }
432 
433 // Get the size in bytes of this frame (rough estimate)
435 {
436  int64_t total_bytes = 0;
437  if (image) {
438  total_bytes += static_cast<int64_t>(
439  width * height * sizeof(char) * 4);
440  }
441  if (audio) {
442  // approximate audio size (sample rate / 24 fps)
443  total_bytes += (sample_rate / 24.0) * sizeof(float);
444  }
445 
446  // return size of this frame
447  return total_bytes;
448 }
449 
450 // Get pixel data (as packets)
451 const unsigned char* Frame::GetPixels()
452 {
453  // Check for blank image
454  if (!image)
455  // Fill with black
456  AddColor(width, height, color);
457 
458  // Return array of pixel packets
459  return image->constBits();
460 }
461 
462 // Get pixel data (for only a single scan-line)
463 const unsigned char* Frame::GetPixels(int row)
464 {
465  // Check for blank image
466  if (!image)
467  // Fill with black
468  AddColor(width, height, color);
469 
470  // Return array of pixel packets
471  return image->constScanLine(row);
472 }
473 
474 // Check a specific pixel color value (returns True/False)
475 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
476  int col_pos = col * 4; // Find column array position
477  if (!image || row < 0 || row >= (height - 1) ||
478  col_pos < 0 || col_pos >= (width - 1) ) {
479  // invalid row / col
480  return false;
481  }
482  // Check pixel color
483  const unsigned char* pixels = GetPixels(row);
484  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
485  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
486  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
487  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
488  // Pixel color matches successfully
489  return true;
490  } else {
491  // Pixel color does not match
492  return false;
493  }
494 }
495 
496 // Set Pixel Aspect Ratio
497 void Frame::SetPixelRatio(int num, int den)
498 {
499  pixel_ratio.num = num;
500  pixel_ratio.den = den;
501 }
502 
503 // Set frame number
504 void Frame::SetFrameNumber(int64_t new_number)
505 {
506  number = new_number;
507 }
508 
509 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
510 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
511 {
512  // Get the total # of samples for the previous frame, and the current frame (rounded)
513  double fps_rate = fps.Reciprocal().ToDouble();
514 
515  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
516  double previous_samples = (sample_rate * fps_rate) * (number - 1);
517  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
518  previous_samples -= previous_samples_remainder;
519 
520  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
521  double total_samples = (sample_rate * fps_rate) * number;
522  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
523  total_samples -= total_samples_remainder;
524 
525  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
526  // be evenly divided into frames, so each frame can have have different # of samples.
527  int samples_per_frame = round(total_samples - previous_samples);
528  if (samples_per_frame < 0)
529  samples_per_frame = 0;
530  return samples_per_frame;
531 }
532 
533 // Calculate the # of samples per video frame (for the current frame number)
534 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
535 {
536  return GetSamplesPerFrame(number, fps, sample_rate, channels);
537 }
538 
539 // Get height of image
541 {
542  return height;
543 }
544 
545 // Get height of image
547 {
548  return width;
549 }
550 
551 // Get the original sample rate of this frame's audio data
553 {
554  return sample_rate;
555 }
556 
557 // Get the original sample rate of this frame's audio data
559 {
560  return channel_layout;
561 }
562 
563 
564 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
565 void Frame::Save(std::string path, float scale, std::string format, int quality)
566 {
567  // Get preview image
568  std::shared_ptr<QImage> previewImage = GetImage();
569 
570  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
571  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
572  {
573  // Resize to fix DAR
574  previewImage = std::make_shared<QImage>(previewImage->scaled(
575  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
576  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
577  }
578 
579  // scale image if needed
580  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
581  {
582  // Resize image
583  previewImage = std::make_shared<QImage>(previewImage->scaled(
584  previewImage->size().width() * scale, previewImage->size().height() * scale,
585  Qt::KeepAspectRatio, Qt::SmoothTransformation));
586  }
587 
588  // Save image
589  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
590 }
591 
592 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
593 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
594  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
595 
596  // Create blank thumbnail image & fill background color
597  auto thumbnail = std::make_shared<QImage>(
598  new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
599  thumbnail->fill(QColor(QString::fromStdString(background_color)));
600 
601  // Create painter
602  QPainter painter(thumbnail.get());
603  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
604 
605  // Get preview image
606  std::shared_ptr<QImage> previewImage = GetImage();
607 
608  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
609  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
610  {
611  // Calculate correct DAR (display aspect ratio)
612  int aspect_width = previewImage->size().width();
613  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
614 
615  // Resize to fix DAR
616  previewImage = std::make_shared<QImage>(previewImage->scaled(
617  aspect_width, aspect_height,
618  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
619  }
620 
621  // Resize frame image
622  if (ignore_aspect)
623  // Ignore aspect ratio
624  previewImage = std::make_shared<QImage>(previewImage->scaled(
625  new_width, new_height,
626  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
627  else
628  // Maintain aspect ratio
629  previewImage = std::make_shared<QImage>(previewImage->scaled(
630  new_width, new_height,
631  Qt::KeepAspectRatio, Qt::SmoothTransformation));
632 
633  // Composite frame image onto background (centered)
634  int x = (new_width - previewImage->size().width()) / 2.0; // center
635  int y = (new_height - previewImage->size().height()) / 2.0; // center
636  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
637 
638 
639  // Create transform and rotate (if needed)
640  QTransform transform;
641  float origin_x = previewImage->width() / 2.0;
642  float origin_y = previewImage->height() / 2.0;
643  transform.translate(origin_x, origin_y);
644  transform.rotate(rotate);
645  transform.translate(-origin_x,-origin_y);
646  painter.setTransform(transform);
647 
648  // Draw image onto QImage
649  painter.drawImage(x, y, *previewImage);
650 
651 
652  // Overlay Image (if any)
653  if (overlay_path != "") {
654  // Open overlay
655  auto overlay = std::make_shared<QImage>();
656  overlay->load(QString::fromStdString(overlay_path));
657 
658  // Set pixel format
659  overlay = std::make_shared<QImage>(
660  overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
661 
662  // Resize to fit
663  overlay = std::make_shared<QImage>(overlay->scaled(
664  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
665 
666  // Composite onto thumbnail
667  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
668  painter.drawImage(0, 0, *overlay);
669  }
670 
671 
672  // Mask Image (if any)
673  if (mask_path != "") {
674  // Open mask
675  auto mask = std::make_shared<QImage>();
676  mask->load(QString::fromStdString(mask_path));
677 
678  // Set pixel format
679  mask = std::make_shared<QImage>(
680  mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
681 
682  // Resize to fit
683  mask = std::make_shared<QImage>(mask->scaled(
684  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
685 
686  // Negate mask
687  mask->invertPixels();
688 
689  // Get pixels
690  unsigned char *pixels = (unsigned char *) thumbnail->bits();
691  const unsigned char *mask_pixels = (const unsigned char *) mask->constBits();
692 
693  // Convert the mask image to grayscale
694  // Loop through pixels
695  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
696  {
697  // Get the RGB values from the pixel
698  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
699  int Frame_Alpha = pixels[byte_index + 3];
700  int Mask_Value = constrain(Frame_Alpha - gray_value);
701 
702  // Set all alpha pixels to gray value
703  pixels[byte_index + 3] = Mask_Value;
704  }
705  }
706 
707 
708  // End painter
709  painter.end();
710 
711  // Save image
712  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
713 }
714 
715 // Constrain a color value from 0 to 255
716 int Frame::constrain(int color_value)
717 {
718  // Constrain new color from 0 to 255
719  if (color_value < 0)
720  color_value = 0;
721  else if (color_value > 255)
722  color_value = 255;
723 
724  return color_value;
725 }
726 
727 void Frame::AddColor(int new_width, int new_height, std::string new_color)
728 {
729  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
730  // Update parameters
731  width = new_width;
732  height = new_height;
733  color = new_color;
734  AddColor(QColor(QString::fromStdString(new_color)));
735 }
736 
737 // Add (or replace) pixel data to the frame (based on a solid color)
738 void Frame::AddColor(const QColor& new_color)
739 {
740  // Create new image object, and fill with pixel data
741  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
742  image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
743 
744  // Fill with solid color
745  image->fill(new_color);
746  has_image_data = true;
747 }
748 
749 // Add (or replace) pixel data to the frame
751  int new_width, int new_height, int bytes_per_pixel,
752  QImage::Format type, const unsigned char *pixels_)
753 {
754  if (has_image_data) {
755  // Delete the previous QImage
756  image.reset();
757  }
758 
759  // Create new image object from pixel data
760  auto new_image = std::make_shared<QImage>(
761  pixels_,
762  new_width, new_height,
763  new_width * bytes_per_pixel,
764  type,
765  (QImageCleanupFunction) &openshot::cleanUpBuffer,
766  (void*) pixels_
767  );
768  AddImage(new_image);
769 }
770 
771 // Add (or replace) pixel data to the frame
772 void Frame::AddImage(std::shared_ptr<QImage> new_image)
773 {
774  // Ignore blank images
775  if (!new_image)
776  return;
777 
778  // assign image data
779  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
780  image = new_image;
781 
782  // Always convert to Format_RGBA8888_Premultiplied (if different)
783  if (image->format() != QImage::Format_RGBA8888_Premultiplied)
784  *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
785 
786  // Update height and width
787  width = image->width();
788  height = image->height();
789  has_image_data = true;
790 }
791 
792 // Add (or replace) pixel data to the frame (for only the odd or even lines)
793 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
794 {
795  // Ignore blank new_image
796  if (!new_image)
797  return;
798 
799  // Check for blank source image
800  if (!image) {
801  // Replace the blank source image
802  AddImage(new_image);
803 
804  } else {
805  // Ignore image of different sizes or formats
806  bool ret=false;
807  if (image == new_image || image->size() != new_image->size()) {
808  ret = true;
809  }
810  else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
811  new_image = std::make_shared<QImage>(
812  new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
813  }
814  if (ret) {
815  return;
816  }
817 
818  // Get the frame's image
819  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
820  unsigned char *pixels = image->bits();
821  const unsigned char *new_pixels = new_image->constBits();
822 
823  // Loop through the scanlines of the image (even or odd)
824  int start = 0;
825  if (only_odd_lines)
826  start = 1;
827 
828  for (int row = start; row < image->height(); row += 2) {
829  int offset = row * image->bytesPerLine();
830  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
831  }
832 
833  // Update height and width
834  height = image->height();
835  width = image->width();
836  has_image_data = true;
837  }
838 }
839 
840 
841 // Resize audio container to hold more (or less) samples and channels
842 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
843 {
844  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
845 
846  // Resize JUCE audio buffer
847  audio->setSize(channels, length, true, true, false);
848  channel_layout = layout;
849  sample_rate = rate;
850 
851  // Calculate max audio sample added
852  max_audio_sample = length;
853 }
854 
855 // Add audio samples to a specific channel
856 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
857  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
858 
859  // Clamp starting sample to 0
860  int destStartSampleAdjusted = max(destStartSample, 0);
861 
862  // Extend audio container to hold more (or less) samples and channels.. if needed
863  int new_length = destStartSampleAdjusted + numSamples;
864  int new_channel_length = audio->getNumChannels();
865  if (destChannel >= new_channel_length)
866  new_channel_length = destChannel + 1;
867  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
868  audio->setSize(new_channel_length, new_length, true, true, false);
869 
870  // Clear the range of samples first (if needed)
871  if (replaceSamples)
872  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
873 
874  // Add samples to frame's audio buffer
875  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
876  has_audio_data = true;
877 
878  // Calculate max audio sample added
879  if (new_length > max_audio_sample)
880  max_audio_sample = new_length;
881 }
882 
883 // Apply gain ramp (i.e. fading volume)
884 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
885 {
886  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
887 
888  // Apply gain ramp
889  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
890 }
891 
892 // Get pointer to Magick++ image object
893 std::shared_ptr<QImage> Frame::GetImage()
894 {
895  // Check for blank image
896  if (!image)
897  // Fill with black
898  AddColor(width, height, color);
899 
900  return image;
901 }
902 
903 #ifdef USE_OPENCV
904 
905 // Convert Qimage to Mat
906 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
907 
908  cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
909  cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
910  int from_to[] = { 0,0, 1,1, 2,2 };
911  cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
912  cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
913  return mat2;
914 }
915 
916 // Get pointer to OpenCV image object
918 {
919  // Check for blank image
920  if (!image)
921  // Fill with black
922  AddColor(width, height, color);
923 
924  // if (imagecv.empty())
925  // Convert Qimage to Mat
926  imagecv = Qimage2mat(image);
927 
928  return imagecv;
929 }
930 
931 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
932  cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
933  QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
934 
935  std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
936 
937  // Always convert to RGBA8888 (if different)
938  if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
939  *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
940 
941  return imgIn;
942 }
943 
944 // Set pointer to OpenCV image object
945 void Frame::SetImageCV(cv::Mat _image)
946 {
947  imagecv = _image;
948  image = Mat2Qimage(_image);
949 }
950 #endif
951 
952 // Play audio samples for this frame
954 {
955  // Check if samples are present
956  if (!GetAudioSamplesCount())
957  return;
958 
959  juce::AudioDeviceManager deviceManager;
960  juce::String error = deviceManager.initialise (
961  0, /* number of input channels */
962  2, /* number of output channels */
963  0, /* no XML settings.. */
964  true /* select default device on failure */);
965 
966  // Output error (if any)
967  if (error.isNotEmpty()) {
968  cout << "Error on initialise(): " << error << endl;
969  }
970 
971  juce::AudioSourcePlayer audioSourcePlayer;
972  deviceManager.addAudioCallback (&audioSourcePlayer);
973 
974  std::unique_ptr<AudioBufferSource> my_source;
975  my_source.reset (new AudioBufferSource (audio.get()));
976 
977  // Create TimeSliceThread for audio buffering
978  juce::TimeSliceThread my_thread("Audio buffer thread");
979 
980  // Start thread
981  my_thread.startThread();
982 
983  juce::AudioTransportSource transport1;
984  transport1.setSource (my_source.get(),
985  5000, // tells it to buffer this many samples ahead
986  &my_thread,
987  (double) sample_rate,
988  audio->getNumChannels()); // sample rate of source
989  transport1.setPosition (0);
990  transport1.setGain(1.0);
991 
992 
993  // Create MIXER
994  juce::MixerAudioSource mixer;
995  mixer.addInputSource(&transport1, false);
996  audioSourcePlayer.setSource (&mixer);
997 
998  // Start transports
999  transport1.start();
1000 
1001  while (transport1.isPlaying())
1002  {
1003  cout << "playing" << endl;
1004  std::this_thread::sleep_for(std::chrono::seconds(1));
1005  }
1006 
1007  cout << "DONE!!!" << endl;
1008 
1009  transport1.stop();
1010  transport1.setSource (0);
1011  audioSourcePlayer.setSource (0);
1012  my_thread.stopThread(500);
1013  deviceManager.removeAudioCallback (&audioSourcePlayer);
1014  deviceManager.closeAudioDevice();
1015  deviceManager.removeAllChangeListeners();
1016  deviceManager.dispatchPendingMessages();
1017 
1018  cout << "End of Play()" << endl;
1019 
1020 
1021 }
1022 
1023 // Add audio silence
1024 void Frame::AddAudioSilence(int numSamples)
1025 {
1026  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
1027 
1028  // Resize audio container
1029  audio->setSize(channels, numSamples, false, true, false);
1030  audio->clear();
1031  has_audio_data = true;
1032 
1033  // Calculate max audio sample added
1034  if (numSamples > max_audio_sample)
1035  max_audio_sample = numSamples;
1036 }
int GetWidth()
Get height of image.
Definition: Frame.cpp:546
int num
Numerator for the fraction.
Definition: Fraction.h:32
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:422
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:727
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:593
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:842
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:931
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:90
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:258
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:451
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:953
Header file for AudioBufferSource class.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:87
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:856
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:120
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:117
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:78
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:534
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:251
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:321
Header file for Frame class.
void SetBuffer(juce::AudioBuffer< float > *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:314
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:504
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:917
Header file for AudioResampler class.
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1024
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:118
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:367
This class represents a fraction.
Definition: Fraction.h:30
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:750
Frame()
Constructor - blank frame.
Definition: Frame.cpp:59
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:884
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:268
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:906
Header file for QtUtilities (compatibiity overlay)
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:497
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:412
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:558
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:893
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:159
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:434
virtual ~Frame()
Destructor.
Definition: Frame.cpp:110
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition: Frame.h:116
juce::AudioBuffer< float > * GetResampledBuffer()
Get the resampled audio buffer.
int den
Denominator for the fraction.
Definition: Fraction.h:33
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:301
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:945
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition: Frame.cpp:428
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:119
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:565
int GetHeight()
Get height of image.
Definition: Frame.cpp:540
int SampleRate()
Get the original sample rate of this frame&#39;s audio data.
Definition: Frame.cpp:552
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:475