From dcec22ae879a6670b6af8aa2b77d2734380e61cb Mon Sep 17 00:00:00 2001 From: JP+ <63192177+joachimpoutaraud@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:43:45 +0100 Subject: [PATCH] Add a `return_array` parameter #294 --- musicalgestures/__init__.py | 48 ++++++++++++++++++++++++++++++---- musicalgestures/_grid.py | 39 +++++++++++++++++---------- musicalgestures/_input_test.py | 9 ++++++- musicalgestures/_utils.py | 7 +++-- 4 files changed, 79 insertions(+), 24 deletions(-) diff --git a/musicalgestures/__init__.py b/musicalgestures/__init__.py index 0627c3a..b274fc8 100644 --- a/musicalgestures/__init__.py +++ b/musicalgestures/__init__.py @@ -25,6 +25,9 @@ class MgVideo(MgAudio): def __init__( self, filename, + array=None, + fps=None, + path=None, # Video parameters filtertype='Regular', thresh=0.05, @@ -50,6 +53,9 @@ def __init__( Args: filename (str): Path to the video file. + array (np.ndarray, optional): Generates an MgVideo object from a video array. Defauts to None. + fps (float, optional): The frequency at which consecutive images from the video array are captured or displayed. Defauts to None. + path (str, optional): Path to save the output video file generated from a video array. Defaults to None. filtertype (str, optional): The `filtertype` parameter for the `motion()` method. `Regular` turns all values below `thresh` to 0. `Binary` turns all values below `thresh` to 0, above `thresh` to 1. `Blob` removes individual pixels with erosion method. Defaults to 'Regular'. thresh (float, optional): The `thresh` parameter for the `motion()` method. Eliminates pixel values less than given threshold. A number in the range of 0 to 1. Defaults to 0.05. starttime (int or float, optional): Trims the video from this start time (s). Defaults to 0. @@ -64,13 +70,16 @@ def __init__( crop (str, optional): If 'manual', opens a window displaying the first frame of the input video file, where the user can draw a rectangle to which cropping is applied. If 'auto' the cropping function attempts to determine the area of significant motion and applies the cropping to that area. Defaults to 'None'. keep_all (bool, optional): If True, preserves an output video file after each used preprocessing stage. Defaults to False. returned_by_process (bool, optional): This parameter is only for internal use, do not use it. Defaults to False. - + sr (int, optional): Sampling rate of the audio file. Defaults to 22050. n_fft (int, optional): Length of the FFT window. Defaults to 2048. hop_length (int, optional): Number of samples between successive frames. Defaults to 512. """ self.filename = filename + self.array = array + self.fps = fps + self.path = path # Name of file without extension (only-filename) self.of = os.path.splitext(self.filename)[0] self.fex = os.path.splitext(self.filename)[1] @@ -94,9 +103,16 @@ def __init__( self.sr = sr self.n_fft = n_fft self.hop_length = hop_length - + self.test_input() - self.get_video() + + if all(arg is not None for arg in [self.array, self.fps]): + if self.path is None: + self.path = os.getcwd() + self.from_numpy(self.array, self.fps, self.path) + else: + self.get_video() + self.info() self.flow = Flow(self, self.filename, self.color, self.has_audio) @@ -124,7 +140,7 @@ def __init__( def test_input(self): """Gives feedback to user if initialization from input went wrong.""" - mg_input_test(self.filename, self.filtertype, self.thresh, self.starttime, self.endtime, self.blur, self.skip, self.frames) + mg_input_test(self.filename, self.array, self.fps, self.filtertype, self.thresh, self.starttime, self.endtime, self.blur, self.skip, self.frames) def info(self, type='video'): """Retrieves the information related to video, audio and format.""" @@ -192,6 +208,8 @@ def average(self, filename=None, normalize=True, target_name=None, overwrite=Fal ################################################## def numpy(self): + import subprocess + "Pipe all video frames from FFmpeg to numpy array" # Define ffmpeg command and pipe it cmd = ['ffmpeg', '-y', '-i', self.filename] @@ -205,7 +223,27 @@ def numpy(self): except ValueError: pass - return array + return array, self.fps + + def from_numpy(self, array, fps, output_path): + import subprocess + + # enforce avi + of, fex = os.path.splitext(self.filename) + self.filename = of + '.avi' + output_path = os.path.join(output_path, self.filename) + process = None + for frame in array: + if process is None: + cmd =['ffmpeg', '-y', '-s', '%dx%d' % (frame.shape[1], frame.shape[0]), '-r', str(fps), + '-c:v', 'rawvideo', '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-i', '-', output_path] + process = subprocess.Popen(cmd, stdin=subprocess.PIPE) + process.stdin.write(frame.tostring()) + process.stdin.close() + process.wait() + + # Save generated musicalgestures video as the video of the parent MgVideo + return self.get_video() class Examples: diff --git a/musicalgestures/_grid.py b/musicalgestures/_grid.py index 45b35a7..fd7587f 100644 --- a/musicalgestures/_grid.py +++ b/musicalgestures/_grid.py @@ -1,10 +1,9 @@ -import os +import os, subprocess import cv2 -import musicalgestures -import matplotlib.pyplot as plt +import numpy as np from musicalgestures._utils import MgImage, generate_outfilename, ffmpeg_cmd, get_length -def mg_grid(self, height=300, rows=3, cols=3, padding=0, margin=0, target_name=None, overwrite=False): +def mg_grid(self, height=300, rows=3, cols=3, padding=0, margin=0, target_name=None, overwrite=False, return_array=False): """ Generates frame strip video preview using ffmpeg. @@ -16,6 +15,7 @@ def mg_grid(self, height=300, rows=3, cols=3, padding=0, margin=0, target_name=N margin (int, optional): Margin size for the grid. Defaults to 0. target_name ([type], optional): Target output name for the grid image. Defaults to None. overwrite (bool, optional): Whether to allow overwriting existing files or to automatically increment target filenames to avoid overwriting. Defaults to False. + return_array (bool, optional): Whether to return an array of not. If set to False the function writes the grid image to disk. Defaults to False. Returns: MgImage: An MgImage object referring to the internal grid image. @@ -36,13 +36,24 @@ def mg_grid(self, height=300, rows=3, cols=3, padding=0, margin=0, target_name=N nth_frame = int(nb_frames / (rows*cols)) # Define the grid specifications - grid = f"select=not(mod(n\,{nth_frame})),scale=-1:{height},tile={cols}x{rows}:padding={padding}:margin={margin}" - - # Declare the ffmpeg command - cmd = ['ffmpeg', '-i', self.filename, '-y', '-frames', '1', '-q:v', '0', '-vf', grid, target_name] - ffmpeg_cmd(cmd, get_length(self.filename), pb_prefix='Rendering video frame grid:') - - # Initialize the MgImage object - img = MgImage(target_name) - - return img \ No newline at end of file + width = int((float(self.width) / self.height) * height) + grid = f"select=not(mod(n\,{nth_frame})),scale={width}:{height},tile={cols}x{rows}:padding={padding}:margin={margin}" + + # Declare the ffmpeg commands + if return_array: + cmd = ['ffmpeg', '-y', '-i', self.filename, '-frames', '1', '-q:v', '0', '-vf', grid] + process = ffmpeg_cmd(cmd, get_length(self.filename), pb_prefix='Rendering video frame grid:', pipe='load') + + if self.color: + array = np.frombuffer(process.stdout, dtype=np.uint8).reshape([-1, height*rows, int(width*cols), 3]) + else: + array = np.frombuffer(process.stdout, dtype=np.uint8).reshape(-1, height*rows, int(width*cols)) + + return array + else: + cmd = ['ffmpeg', '-i', self.filename, '-y', '-frames', '1', '-q:v', '0', '-vf', grid, target_name] + ffmpeg_cmd(cmd, get_length(self.filename), pb_prefix='Rendering video frame grid:') + # Initialize the MgImage object + img = MgImage(target_name) + + return img \ No newline at end of file diff --git a/musicalgestures/_input_test.py b/musicalgestures/_input_test.py index da80524..d2ce97e 100644 --- a/musicalgestures/_input_test.py +++ b/musicalgestures/_input_test.py @@ -15,12 +15,14 @@ def __init__(self, message): self.message = message -def mg_input_test(filename, filtertype, thresh, starttime, endtime, blur, skip, frames): +def mg_input_test(filename, array, fps, filtertype, thresh, starttime, endtime, blur, skip, frames): """ Gives feedback to user if initialization from input went wrong. Args: filename (str): Path to the input video file. + array (np.ndarray, optional): Generates an MgVideo object from a video array. Defauts to None. + fps (float, optional): The frequency at which consecutive images from the video array are captured or displayed. Defauts to None. filtertype (str): 'Regular' turns all values below `thresh` to 0. 'Binary' turns all values below `thresh` to 0, above `thresh` to 1. 'Blob' removes individual pixels with erosion method. thresh (float): A number in the range of 0 to 1. Eliminates pixel values less than given threshold. starttime (int/float): Trims the video from this start time (s). @@ -36,6 +38,11 @@ def mg_input_test(filename, filtertype, thresh, starttime, endtime, blur, skip, filenametest = type(filename) == str if filenametest: + if array is not None: + if fps is None: + msg = 'Please specify frame per second (fps) parameter for generating video from array.' + raise InputError(msg) + if filtertype.lower() not in ['regular', 'binary', 'blob']: msg = 'Please specify a filter type as str: "Regular", "Binary" or "Blob"' raise InputError(msg) diff --git a/musicalgestures/_utils.py b/musicalgestures/_utils.py index 52b8fa3..3d93ebb 100644 --- a/musicalgestures/_utils.py +++ b/musicalgestures/_utils.py @@ -977,8 +977,7 @@ def extract_wav(filename, target_name=None, overwrite=False): print(f'{filename} is already in .wav container.') return filename - cmds = ' '.join(['ffmpeg', '-loglevel', 'quiet', '-y', '-i', wrap_str(filename), "-acodec", - "pcm_s16le", wrap_str(target_name)]) + cmds = ' '.join(['ffmpeg', '-loglevel', 'quiet', '-y', '-i', wrap_str(filename), "-acodec", "pcm_s16le", wrap_str(target_name)]) os.system(cmds) return target_name @@ -1407,13 +1406,13 @@ def ffmpeg_cmd(command, total_time, pb_prefix='Progress', print_cmd=False, strea if pipe == 'read': # Define ffmpeg command and read frame by frame - command = command + ['-f', 'image2pipe', '-pix_fmt', 'bgr24', '-vcodec', 'rawvideo', '-'] + command = command + ['-f', 'image2pipe', '-pix_fmt', 'bgr24', '-vcodec', 'rawvideo', '-preset', 'ultrafast', '-'] process = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=-1) return process elif pipe == 'load': # Define ffmpeg command and load all frames - command = command + ['-f', 'image2pipe', '-pix_fmt', 'bgr24', '-vcodec', 'rawvideo', '-'] + command = command + ['-f', 'image2pipe', '-pix_fmt', 'bgr24', '-vcodec', 'rawvideo', '-preset', 'ultrafast', '-'] process = subprocess.run(command, stdout=subprocess.PIPE, bufsize=-1) return process