thebears/SearchInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YACWC

Browse Source
This commit is contained in:
Ishan S. Patel
2025-09-22 11:07:28 -04:00
parent 47f631fedb
commit aa5ad8327e
9 changed files with 821 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
VectorService/util/embed_scores.py
View File

@@ -134,20 +134,22 @@ def calculate_embedding_score_in_folders(c_dirs, threshold, query = None, query_
# kwargs = [{'c_dir':x, 'threshold':threshold, 'query': query} for x in c_dirs]
args = [(x, threshold, query, None, logger, redis_key) for x in c_dirs]
logger.info(f"CALCULATING FOR {args}")
# logger.info(f"CALCULATING FOR {args}")
with Pool(processes=8) as pool:
out = pool.starmap(calculate_embedding_score_in_folder, args)
logger.info(f"DONE CALCULATING FOR {args}")
# logger.info(f"DONE CALCULATING FOR {args}")
for x in out:
cache_files = list();
for x, cache_file_loc in out:
try:
result_list.extend(x['videos'])
cache_files.append(cache_file_loc);
except Exception as e:
print(e, x)
return {'videos':result_list}
return {'videos':result_list, 'cache_file_locs': cache_files}
def collapse_scores_to_maxmin_avg(folder_scores):
@@ -215,7 +217,7 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v
logger.info(f"LOADED EMBEDDING SCORE FROM CACHE {cache_file_loc}")
message = {'task':'SCORE_CALC_IN_FOLDER_DONE', 'when': str(c_dir), 'time': dt.datetime.now().timestamp(), 'precomputed': True}
r.rpush(redis_key, json.dumps(message))
return video_json_info
return (video_json_info, cache_file_loc)
else:
logger.info(f"CACHE FILE IS OLD, DELETING VEC REP FILE AND RECREATING {cache_file_loc}")
os.remove( get_vec_rep_file_loc(c_dir))
@@ -288,8 +290,8 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v
with open(cache_file_loc, 'wb') as f:
logger.info(f"WRITING EMBEDDING SCORE TO CACHE {cache_file_loc}")
pickle.dump(to_write, f)
return to_write
logger.info(f"SAVED EMBEDDING SCORE TO CACHE {cache_file_loc}")
return (to_write, cache_file_loc)
def get_matching_file_given_filename(web_name, folder_scores):