import os import random from multiprocessing import Pool import sys sys.path.append('/home/thebears/Seafile/Designs/ML') from trim_video import trim_video #rtpath = '/srv/ftp/hummingbird/2021' rtpath = sys.argv[1] have_json = set() fnames = set() for di,_, fns in os.walk(rtpath): for fn in fns: if fn.endswith('.json') and 'trimmed' not in fn: have_json.add(os.path.join(di,fn)) files_to_score = have_json def try_catch_chunk(jsons): try: if not isinstance(jsons, list): jsons = [jsons] for x in jsons: trim_video(x) except Exception as e: print(e) lst = list(files_to_score) n = 25 chunks = [lst[i:i + n] for i in range(0, len(lst), n)] # %% if __name__ == '__main__': with Pool(8) as p: output = p.map(try_catch_chunk,chunks) # output = p.map(score_video,chunks)