thebears/pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YACWC

Browse Source
This commit is contained in:
Ishan S. Patel
2025-08-28 12:38:34 -04:00
parent 571500b291
commit 028149b786
1 changed files with 19 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
00_watch_for_new_videos/load_missed.py
View File

@@ -26,17 +26,26 @@ default_dir_watch = settings.dir_watch
publish = kwq.producer.send publish = kwq.producer.send
topic = kwq.TOPICS.videos_to_score_detection topic = kwq.TOPICS.videos_to_score_detection
necessary_extensions = [
'.oclip_embeds.npz',
'.json',
'.db_has_sl2_embeds_part' ]
def decide_to_put_in_queue(file_name): def decide_to_put_in_queue(file_name):
disqualifiers = list() disqualifiers = list()
disqualifiers.append(not os.path.exists(file_name))
disqualifiers.append("_reduced" in file_name) disqualifiers.append("_reduced" in file_name)
disqualifiers.append("_trimmed" in file_name) disqualifiers.append("_trimmed" in file_name)
disqualified = any(disqualifiers) disqualified = any(disqualifiers)
if file_name.endswith(".mp4") and not disqualified: if file_name.endswith(".mp4") and not disqualified:
return True rt_path, _ = os.path.splitext(file_name)
all_exist = all([os.path.exists( rt_path + ext) for ext in necessary_extensions])
return (not all_exist)
else: else:
return False return False
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir) sys.path.append(parent_dir)
def place_file_in_queue(filef): def place_file_in_queue(filef):
@@ -60,7 +69,7 @@ def place_file_in_queue(filef):
wq.publish(wq.TOPICS.ml_vision_to_score, filef) wq.publish(wq.TOPICS.ml_vision_to_score, filef)
def add_files_to_queue(paths): def add_files_to_queue(paths, dry_run = False):
queued = set() queued = set()
for rt in paths: for rt in paths:
for root, dirs, files in os.walk(rt): for root, dirs, files in os.walk(rt):
@@ -71,7 +80,10 @@ def add_files_to_queue(paths):
for x in queued: for x in queued:
place_file_in_queue(x) if dry_run:
print('DRY RUN, CANDIDATES FOR QUEUE: ' + x)
else:
place_file_in_queue(x)
# %% # %%
@@ -81,13 +93,14 @@ if __name__ == "__main__":
) )
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=()) parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
args, _ = parser.parse_known_args() args, _ = parser.parse_known_args()
paths = args.paths paths = args.paths
dry_run = args.dry_run
if len(paths) == 0: if len(paths) == 0:
paths = default_dir_watch paths = default_dir_watch
add_files_to_queue(paths) add_files_to_queue(paths, dry_run = dry_run)