From b9089ffe9cace1f71eb0a0dca7e8aedf855b0fc4 Mon Sep 17 00:00:00 2001 From: Balint Laczko Date: Mon, 28 Jun 2021 22:30:32 +0200 Subject: [PATCH] fixed minor bugs when using different numbers of processes - mg_motion_mp will now produce _exactly_ the same results with any number of processes (tested from 1 to 12). - added num_processes parameter #213 --- musicalgestures/_motionvideo_mp_render.py | 31 ++++---------- musicalgestures/_motionvideo_mp_run.py | 51 +++++++++++++++++------ 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/musicalgestures/_motionvideo_mp_render.py b/musicalgestures/_motionvideo_mp_render.py index 285560c..3e3b849 100644 --- a/musicalgestures/_motionvideo_mp_render.py +++ b/musicalgestures/_motionvideo_mp_render.py @@ -125,7 +125,7 @@ def mg_motion_mp(args): motion_frame_rgb = motion_frame if save_video: - # if this is not the first process (rendering the start of the video) then don't save the first frame of the output (it'll always be black). + # if this is not the first process (rendering the start of the video) then drop the first frame of the output (it'll always be black). if process_id != 0 and ii > 0: if inverted_motionvideo: out.write(cv2.bitwise_not( @@ -185,9 +185,7 @@ def mg_motion_mp(args): def run_pool(func, args, numprocesses): pool = multiprocessing.Pool(numprocesses) - # results = pool.map(func, args) pool.map(func, args) - # return results def calc_frame_groups(framecount, num_cores): @@ -204,18 +202,6 @@ def calc_frame_groups(framecount, num_cores): return groups -def testhogfunc(process_id): - limit = 20 - count = 0 - while count < limit: - print(process_id, count) - futyi = 0 - for i in range(10_000_000): - futyi += 1 - count += 1 - return process_id, count - - def bool_from_str(boolstring): return True if boolstring == "True" else False @@ -240,6 +226,7 @@ def bool_from_str(boolstring): parser.add_argument('save_data', metavar='save_data', type=str, help='save_data') parser.add_argument('save_motiongrams', metavar='save_motiongrams', type=str, help='save_motiongrams') parser.add_argument('save_video', metavar='save_video', type=str, help='save_video') + parser.add_argument('num_processes', metavar='num_processes', type=str, help='num_processes') args = parser.parse_args() @@ -257,21 +244,19 @@ def bool_from_str(boolstring): fps, width, height, length = int(float(args.fps)), int(float(args.width)), int(float(args.height)), int(float(args.length)) color, filtertype, thresh, blur, kernel_size = bool_from_str(args.color), args.filtertype, float(args.thresh), args.blur, int(float(args.kernel_size)) inverted_motionvideo, inverted_motiongram, equalize_motiongram = bool_from_str(args.inverted_motionvideo), bool_from_str(args.inverted_motiongram), bool_from_str(args.equalize_motiongram) - save_data, save_motiongrams, save_video = bool_from_str(args.save_data), bool_from_str(args.save_motiongrams), bool_from_str(args.save_video) + save_data, save_motiongrams, save_video = bool_from_str(args.save_data), bool_from_str(args.save_motiongrams), bool_from_str(args.save_video) + num_processes = multiprocessing.cpu_count() if int(float(args.num_processes)) < 1 else min(int(float(args.num_processes)), multiprocessing.cpu_count()) - numprocessors = multiprocessing.cpu_count() - frame_groups = calc_frame_groups(length, numprocessors) + frame_groups = calc_frame_groups(length, num_processes) feed_args = [] - for i in range(numprocessors): + for i in range(num_processes): start_frame, num_frames = frame_groups[i] initargs = [target_folder, of, fex, fps, width, height, length, color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data, save_motiongrams, save_video, start_frame, num_frames, i, client] # client.sendall(bytes(str(initargs), 'utf-8')) feed_args.append(initargs) - # results = run_pool(mg_motion_mp, feed_args, numprocessors) - run_pool(mg_motion_mp, feed_args, numprocessors) + run_pool(mg_motion_mp, feed_args, num_processes) - client.close() - # print("Closed socket client.") \ No newline at end of file + client.close() \ No newline at end of file diff --git a/musicalgestures/_motionvideo_mp_run.py b/musicalgestures/_motionvideo_mp_run.py index ffe2d41..7cc776b 100644 --- a/musicalgestures/_motionvideo_mp_run.py +++ b/musicalgestures/_motionvideo_mp_run.py @@ -33,7 +33,8 @@ def mg_motion_mp( target_name_data=None, target_name_mgx=None, target_name_mgy=None, - overwrite=False): + overwrite=False, + num_processes=-1): of, fex = self.of, self.fex @@ -64,7 +65,7 @@ def mg_motion_mp( save_data_feed = save_data or save_plot - command = [pythonkw, pyfile, temp_folder, of_feed, fex, self.fps, self.width, self.height, self.length, self.color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data_feed, save_motiongrams, save_video] + command = [pythonkw, pyfile, temp_folder, of_feed, fex, self.fps, self.width, self.height, self.length, self.color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data_feed, save_motiongrams, save_video, num_processes] command = [str(item) for item in command] # print() # print(command) @@ -108,28 +109,54 @@ def mg_motion_mp( # print("organizing results...") results = os.listdir(temp_folder) time_files = [temp_folder + file for file in results if file.startswith("time")] + time_files.sort() com_files = [temp_folder + file for file in results if file.startswith("com")] + com_files.sort() qom_files = [temp_folder + file for file in results if file.startswith("qom")] + qom_files.sort() gramx_files = [temp_folder + file for file in results if file.startswith("gramx")] + gramx_files.sort() gramy_files = [temp_folder + file for file in results if file.startswith("gramy")] + gramy_files.sort() video_files = [temp_folder + file for file in results if file.endswith("avi")] + video_files.sort() gramx, gramy, time, com, qom = None, None, None, None, None if save_motiongrams: # load gramx - for idx, item in enumerate(gramx_files): - if idx == 0: - gramx = np.load(item) - else: - gramx = np.append(gramx, np.load(item)[1:-1], axis=0) + # if we only used a single chunk, load everything + if len(gramx_files) == 1: + gramx = np.load(gramx_files[0]) + # or in case there were multiple chunks... + else: + for idx, item in enumerate(gramx_files): + if idx == 0: + # do not drop first row in first chunk + gramx = np.load(item)[:-1] + elif idx == len(gramy_files) - 1: + # do not drop the last row in last chunk + gramx = np.append(gramx, np.load(item)[1:], axis=0) + else: + # else drop first and last rows from chunk + gramx = np.append(gramx, np.load(item)[1:-1], axis=0) # load gramy - for idx, item in enumerate(gramy_files): - if idx == 0: - gramy = np.load(item) - else: - gramy = np.append(gramy, np.load(item)[:, 1:], axis=1) + # if we only used a single chunk, load everything + if len(gramy_files) == 1: + gramy = np.load(gramy_files[0]) + # or in case there were multiple chunks... + else: + for idx, item in enumerate(gramy_files): + if idx == 0: + # do not drop first column in first chunk + gramy = np.load(item)[:, :-1] + elif idx == len(gramy_files) - 1: + # do not drop the last column in last chunk + gramy = np.append(gramy, np.load(item)[:, 1:], axis=1) + else: + # else drop first and last columns from chunk + gramy = np.append(gramy, np.load(item)[:, 1:-1], axis=1) if self.color == False: # Normalize before converting to uint8 to keep precision