YACWC

VectorService/util/embed_scores.py

@@ -134,20 +134,22 @@ def calculate_embedding_score_in_folders(c_dirs, threshold, query = None, query_
 #    kwargs = [{'c_dir':x, 'threshold':threshold, 'query': query}  for x in c_dirs]
     args = [(x, threshold, query, None, logger, redis_key) for x in c_dirs]
     
-    logger.info(f"CALCULATING FOR {args}")
+    # logger.info(f"CALCULATING FOR {args}")
     with Pool(processes=8) as pool:
         out = pool.starmap(calculate_embedding_score_in_folder, args)
-    logger.info(f"DONE CALCULATING FOR {args}")
+    # logger.info(f"DONE CALCULATING FOR {args}")
 
-    for x in out:
+    cache_files = list();
+    for x, cache_file_loc in out:
         try:
             result_list.extend(x['videos'])
+            cache_files.append(cache_file_loc);
         except Exception as e:
             print(e, x) 
 
 
         
-    return {'videos':result_list}
+    return {'videos':result_list, 'cache_file_locs': cache_files}
 
 
 def collapse_scores_to_maxmin_avg(folder_scores):
@@ -215,7 +217,7 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v
                     logger.info(f"LOADED EMBEDDING SCORE FROM CACHE {cache_file_loc}")
                     message = {'task':'SCORE_CALC_IN_FOLDER_DONE', 'when': str(c_dir), 'time': dt.datetime.now().timestamp(), 'precomputed': True}
                     r.rpush(redis_key, json.dumps(message)) 
-                    return video_json_info
+                    return (video_json_info, cache_file_loc)
                 else:
                     logger.info(f"CACHE FILE IS OLD, DELETING VEC REP FILE AND RECREATING {cache_file_loc}")
                     os.remove( get_vec_rep_file_loc(c_dir))
@@ -288,8 +290,8 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v
     with open(cache_file_loc, 'wb') as f:
         logger.info(f"WRITING EMBEDDING SCORE TO CACHE {cache_file_loc}")
         pickle.dump(to_write, f) 
-
-    return to_write
+    logger.info(f"SAVED EMBEDDING SCORE TO CACHE {cache_file_loc}")
+    return (to_write, cache_file_loc)
 
 
 def get_matching_file_given_filename(web_name, folder_scores):

VectorService/vector_service.py

@@ -82,9 +82,6 @@ async def videos_json(
 
     folder_scores["breaks"] = ES.add_breaks_between_videos(folder_scores)
     folder_scores['videos'] = ES.collapse_scores_to_maxmin_avg(folder_scores)
-
-
-    session["folder_scores"] = folder_scores
     return folder_scores