YACWC

orin/02_remove_without_nuggets/remove_without_nuggets.py

@@ -1,132 +0,0 @@
-#!/usr/bin/python
-import glob
-import argparse
-import os
-import json
-from CommonCode.util import exit_if_not_ipython, get_cset_for_file_matching, is_ipython
-from CommonCode import kwq
-from CommonCode.settings import get_logger, LogColorize
-from kafka import TopicPartition
-from kafka.structs import OffsetAndMetadata
-from pprint import pprint
-
-pfm = LogColorize.remove_without_nuggets
-logger = get_logger(__name__,'/var/log/ml_vision_logs/02_remove_without_nuggets', stdout=True, systemd=False)
-
-input_topic = kwq.TOPICS.videos_scored_detection
-
-client_id='obj_detector_remove_without_nuggets'
-group_id = client_id
-
-consumer = kwq.create_consumer(input_topic, group_id = group_id, client_id = client_id)
-c_part = TopicPartition(input_topic, 0)
-consumer.assign([c_part])
-
-producer = kwq.producer
-
-
-class RESULT_TYPE():
-    NOTHING = 0
-    HAS_OBJS = 1
-    KEPT = 2
-    REMOVED = 3
-    NO_JSON = 4
-
-
-def get_ok_to_delete(file_path):
-    max_look = 5
-    c_path = file_path
-    settings_default = {'class_threshold':10000, 'frames_with_dets_threshold': -10000}
-    for i in range(max_look):
-        c_path  = os.path.abspath(os.path.join(c_path, '..'))
-        test_path = os.path.join(c_path, 'settings.json')
-        if os.path.exists(test_path):
-            with open(test_path, 'r') as rr:
-                settings = json.load(rr)
-                settings_default.update(settings)
-
-    return settings_default
-
-def exec_file_remove_logic(file_path):
-    result = RESULT_TYPE.NOTHING
-    cset = get_cset_for_file_matching(file_path)
-
-    settings = get_ok_to_delete(cset['.mp4'])
-    logger.info(f"EVALUATING LOGIC WITH SETTINGS: "+str(settings))
-    if settings['class_threshold'] > 1:
-        logger.info(f"THRESHOLD SET TO ABOVE 1, SKIPPING: {file_path}")
-        result = RESULT_TYPE.HAS_OBJS
-        return
-    
-    logger.info(f"EXECUTING_LOGIC :{file_path}")
-    json_check = '.json.orin'
-    if '.has_objs' in cset:
-        logger.info(f"HAS_OBJS :{file_path}")
-        result = RESULT_TYPE.HAS_OBJS
-        return result
-
-    
-    if json_check in cset and os.path.exists(cset[ json_check]):
-        logger.info(f"HAS_JSON :{file_path}")    
-        if os.path.getsize(cset[json_check]) == 0:
-            det_data = {'scores':[]}
-        else:
-            with open(cset[json_check],'r') as jj:
-                det_data = json.load(jj)
-
-        thresh = settings['class_threshold']
-        frames_with_dets_thresh = settings['frames_with_dets_threshold']
-
-        all_scores = list()
-        for x in det_data['scores']:
-            sc_in = [y['score'] for y in x['detections']]
-            sc_in.append(0)
-            all_scores.append(max(sc_in))
-        
-        frames_above_thresh = sum([x>thresh for x in all_scores ])
-        num_frames = len(det_data['scores'])
-
-        ratio_with_frames = frames_above_thresh / (num_frames + 1)
-        if ratio_with_frames > frames_with_dets_thresh:
-            cpath = os.path.splitext(cset['.mp4'])[0] + '.has_objs.orin'
-            with open(cpath,'w') as cc:
-                pass
-
-
-            logger.info(f"HAS_OBJECTS_DETECTED_WILL_NOT_REMOVE :{pfm(file_path)}")
-            result = RESULT_TYPE.KEPT
-
-        else:
-            logger.info(f"OBJECTS_NOT_DETECTED_WILL_REMOVE :{pfm(file_path)}")
-            for ff, cf in cset.items():
-                if cf.endswith(json_check) or cf.endswith('.jpg'):
-                    pass
-                else:
-                    logger.info(f"REMOVING :{cf}")
-                    os.remove(cf)
-
-            result = RESULT_TYPE.REMOVED
-
-    else:
-        result = RESULT_TYPE.NO_JSON
-        logger.info(f"NO_JSON_DONT_REMOVE :{pfm(file_path)}")
-
- 
-    return result
-
-
-for msg in consumer:
-    key = msg.key
-    value = msg.value
-
-    purge_reason = exec_file_remove_logic(value['filepath'])
-    d_send = {'value':msg.value, 'key':msg.key}
-    if purge_reason in [RESULT_TYPE.KEPT, RESULT_TYPE.HAS_OBJS]:
-        producer.send(kwq.TOPICS.videos_with_nuggets, **d_send)
-    elif purge_reason == RESULT_TYPE.REMOVED:
-        producer.send(kwq.TOPICS.videos_without_nuggets, **d_send)
-    elif purge_reason == RESULT_TYPE.NO_JSON:
-        producer.send(kwq.TOPICS.videos_no_json, **d_send)
-    
-# %%
- 

orin/03_put_into_vectordb/put_into_vectordb.py

@@ -1,115 +0,0 @@
-import glob
-from pymilvus import MilvusClient
-from pymilvus.client.types import LoadState
-
-import argparse
-import os
-import json
-from CommonCode.util import exit_if_not_ipython, get_cset_for_file_matching, is_ipython
-from CommonCode import kwq
-from CommonCode.settings import get_logger, LogColorize
-from kafka import TopicPartition
-from kafka.structs import OffsetAndMetadata
-from pprint import pprint
-import numpy as np
-pfm = LogColorize.embeds_in_db
-logger = get_logger(__name__,'/var/log/ml_vision_logs/03_put_into_vectordb', stdout=True, systemd=False)
-
-input_topic = kwq.TOPICS.videos_with_nuggets
-
-client_id='embedding_place_in_db_1'
-group_id = client_id
-
-consumer = kwq.create_consumer(input_topic, group_id = group_id, client_id = client_id)
-c_part = TopicPartition(input_topic, 0)
-consumer.assign([c_part])
-
-
-producer = kwq.producer
-model_type = 'ViT-L-16-SigLIP2-512'
-
-def get_db_embed_done_path(vpath):
-    return os.path.splitext(vpath)[0]+'.oclip.orin.indb'
-
-
-#video_file_to_upload='/srv/ftp/ptz/2025/04/14/PTZBackRight_00_20250414063817.mp4'
-
-
-def get_vec_path(vpath):
-    return os.path.splitext(vpath)[0]+'.oclip.orin'
-
-def get_date(vpath):
-    split_entries = os.path.splitext(vpath)[0].split('/')
-    return ''.join(split_entries[-4:-1])
-
-def get_camera_name(vpath):
-    split_entries = os.path.splitext(vpath)[0].split('/')
-    return split_entries[split_entries.index('ftp')+1]
-
-def upload_vector_file(video_file_to_upload):
-    client = MilvusClient(
-        uri="http://localhost:19530"
-    )
-
-    db_done_path = get_db_embed_done_path(video_file_to_upload)
-    if os.path.exists(db_done_path):
-        print('Already exists in DB, skipping upload')
-#        return
-
-    video_file_to_upload = get_vec_path(video_file_to_upload)
-
-    with open(video_file_to_upload,'r') as jj:
-        vecs = json.load(jj)
-
-
-    embeds = [x['score'] for x in vecs['scores']]
-    fr_nums =  [x['frame'] for x in vecs['scores']]
-
-    fname_root = video_file_to_upload.rsplit('/',1)[-1].split('.')[0]
-    fc = fname_root.split('_')[-1]
-
-#    data = list()
-    filepath = video_file_to_upload.replace('/srv/ftp/','').replace('/mergedfs/ftp','').split('.')[-0]
-
-    data_v2 = list()
-    date = get_date(filepath)
-    for embed, frame_num in zip(embeds, fr_nums):
-        fg = '{0:05g}'.format(frame_num)
-        id_num = int(fc+fg)
-        embeds_as_np = np.asarray(embed, dtype=np.float16)
-        to_put_2 = dict(primary_id= id_num, filepath=filepath, frame_number = int(frame_num), so400m=embeds_as_np, date=str(date))
-        data_v2.append(to_put_2)
-
-    cam_name = get_camera_name(video_file_to_upload)
-    client.insert(collection_name = f'nuggets_{cam_name}_so400m_siglip2', data=data_v2)
-    client.close()
-
-    
-    with open(db_done_path,'w') as ff:
-        ff.write("")
-        
-    print(f'Inserting into DB, {video_file_to_upload}')
-
-
-
-for msg in consumer:
-    key = msg.key
-    value = msg.value
-    file_path = value['filepath']
-    success = False
-    try:
-        upload_vector_file(value['filepath'])
-        success = True
-        logger.info(f"SUCCESS_UPLOADING :{pfm(file_path)}")
-    except Exception as e:
-        logger.info(f"ERROR_UPLOADING :{pfm(file_path)} + {e}")
-
-    d_send = {'value':msg.value, 'key':msg.key}
-
-    if success:
-        send_topic = kwq.TOPICS.videos_embedding_in_db
-    else:
-        send_topic = kwq.TOPICS.videos_embedding_in_db_fail
-        
-
-    producer.send(send_topic, **d_send)

vision_v3/00_watch_for_new_videos/00_load_missed.py

@@ -0,0 +1,116 @@
+#!/home/thebears/envs/ml_vision_objdet/bin/python
+import os
+
+import subprocess as sp
+import time
+import psutil
+import stat
+import sys
+import logging
+from watchdog.observers.polling import PollingObserver
+from watchdog.events import FileSystemEventHandler
+from threading import Thread
+from tqdm import tqdm
+from common_code import kwq
+from common_code.settings import get_logger, LogColorize
+import argparse
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
+
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(parent_dir)
+logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
+from common_code import settings
+
+#default_dir_watch = settings.dir_watch_missed
+default_dir_watch = settings.dir_watch
+
+publish = kwq.producer.send
+#topic = kwq.TOPICS.exit_00_videos_to_score_detection
+topic = kwq.TOPICS.enter_60_videos_embed_backfill
+
+necessary_extensions = [
+    '.oclip_embeds.npz',
+    '.detection.npz']
+
+def decide_to_put_in_queue(file_name, force = False):
+    if os.path.getsize(file_name) < 1*1000*1000:
+        return False
+    disqualifiers = list()
+    disqualifiers.append(not  os.path.exists(file_name))
+    disqualifiers.append("_reduced" in file_name)
+    disqualifiers.append("_trimmed" in file_name)
+#    disqualifiers.append('hummingbird' in file_name)
+
+    disqualified = any(disqualifiers)
+    if file_name.endswith(".mp4") and not disqualified:
+        rt_path, _ = os.path.splitext(file_name)
+        if force:
+            all_exist = False
+        else:
+            all_exist = all([os.path.exists( rt_path + ext) for ext in necessary_extensions])
+        return (not all_exist)
+    else:
+        return False
+
+    
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(parent_dir)
+def place_file_in_queue(filef):
+    dirname = os.path.dirname(filef)
+    foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
+    if oct(foldmode) != "0o777":
+        os.system(f"sudo chmod 777 {dirname}")
+
+    logging.info(f"SETTING_PERMISSIONS: {dirname}")
+
+    chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
+    try:
+        
+        os.chmod(filef, 0o777)
+    except Exception as e:
+        pass
+#$        logging.error(e)
+
+    logging.info(f"QUEUE_PUT: {filef}")
+    publish(topic, key=filef, value={"filepath": filef})
+
+
+
+def add_files_to_queue(paths, dry_run = False, force = False):
+    queued = set()
+    for rt in paths:
+        for root, dirs, files in os.walk(rt):
+            for f in files:
+                new_path = os.path.join(root, f)
+                if decide_to_put_in_queue(new_path, force = force):
+                    queued.add(new_path)
+
+
+    for x in tqdm(queued):
+        if dry_run:
+            print('DRY RUN, CANDIDATES FOR QUEUE: ' + x)
+        else:
+            place_file_in_queue(x)
+
+    
+# %%
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Watch and Fix Permissions And Push to Kafka Queue"
+    )
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+    parser.add_argument("--dry-run", action='store_true', help='Dry Run')
+    parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')    
+    args, _ = parser.parse_known_args()
+    paths = args.paths
+    force = args.force
+    dry_run = args.dry_run
+
+    if len(paths) == 0:
+        paths = default_dir_watch
+
+    add_files_to_queue(paths, dry_run = dry_run, force=force)
+
+    

vision_v3/00_watch_for_new_videos/00_watch_and_fix_permissions.py

@@ -8,36 +8,34 @@ import logging
 from watchdog.observers.polling import PollingObserver
 from watchdog.events import FileSystemEventHandler
 from threading import Thread
-from CommonCode import wq
-from CommonCode.util import is_ipython
+from common_code.util import is_ipython
 import queue
-from CommonCode.settings import get_logger, LogColorize
+from common_code.settings import get_logger, LogColorize
 import argparse
-
+from common_code import kwq
 import subprocess
 from collections import deque
 
+
+publish = kwq.producer.send
+topic = kwq.TOPICS.exit_00_videos_to_score_detection
+
+
 pfm = LogColorize.watch_and_fix_permissions
 if not ('__file__' in vars() or '__file__' in globals()):
-    __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/00_watch_for_new_videos/00_watch_and_fix_permissions.py'
 
 parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 sys.path.append(parent_dir)
 
-from common import settings
+from common_code import settings
 default_dir_watch = settings.dir_watch
 
 tasks = queue.Queue()
 logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
 
-logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
+logger.info(pfm(f"Starting file watcher  @ {__file__}"))
 
-try:
-    from CommonCode import kwq
-    publish = kwq.producer.send
-    topic = kwq.TOPICS.exit_00_videos_to_score_detection
-except Exception as e:
-    logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
 
 def decide_to_put_in_queue(file_name):
     disqualifiers = list()
@@ -112,7 +110,7 @@ def process_file(filef):
 
     if decide_to_put_in_queue(filef):
         logging.info(f"QUEUE_PUT: {pfm(filef)}")
-        wq.publish(wq.TOPICS.ml_vision_to_score, filef)
+
         try:
             future = publish(topic, key=filef, value={"filepath": filef})
         except Exception as e:

vision_v3/00_watch_for_new_videos/load_missed_leopards34.py

@@ -0,0 +1,20 @@
+from load_missed_lib import place_file_in_queue
+
+# %%
+import os
+
+rrt = ['/srv/ftp_tcc/', '/mnt/hdd_24tb_1/videos/ftp']
+dir_scan = ['leopards3','leopards4']
+
+
+rt_scans = list()
+
+for r1 in rrt:
+    for d1 in dir_scan:
+        rt_scans.append(os.path.join(r1, d1))
+
+# %%
+for rt_s in rt_scans:
+    for root, dirs, files in os.walk(rt_s):
+        for x in files:
+            print(x)

vision_v3/00_watch_for_new_videos/load_missed_lib.py

@@ -10,9 +10,9 @@ import logging
 from watchdog.observers.polling import PollingObserver
 from watchdog.events import FileSystemEventHandler
 from threading import Thread
-from CommonCode import wq
-from CommonCode import kwq
-from CommonCode.settings import get_logger, LogColorize
+
+from common_code import kwq
+from common_code.settings import get_logger, LogColorize
 import argparse
 if not ('__file__' in vars() or '__file__' in globals()):
     __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
@@ -20,7 +20,7 @@ if not ('__file__' in vars() or '__file__' in globals()):
 parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 sys.path.append(parent_dir)
 logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
-from common import settings
+from common_code import settings
 
 #default_dir_watch = settings.dir_watch_missed
 default_dir_watch = settings.dir_watch
@@ -30,8 +30,7 @@ topic = kwq.TOPICS.exit_00_videos_to_score_detection
 
 necessary_extensions = [
     '.oclip_embeds.npz',
-    '.json',
-    '.db_has_sl2_embeds_part'    ]
+    '.detection.npz']
 
 def decide_to_put_in_queue(file_name):
     if os.path.getsize(file_name) < 1*1000*1000:
@@ -71,7 +70,7 @@ def place_file_in_queue(filef):
 
     logging.info(f"QUEUE_PUT: {filef}")
     publish(topic, key=filef, value={"filepath": filef})
-    wq.publish(wq.TOPICS.ml_vision_to_score, filef)
+
 
 
 def add_files_to_queue(paths, dry_run = False):

vision_v3/00_watch_for_new_videos/test.py

@@ -1,11 +0,0 @@
-from CommonCode import kwq
-publish = kwq.producer.send
-topic = kwq.TOPICS.videos_to_score_detection
-cons = kwq.create_consumer()
-
-# %%
-cons.subscribe([topic])
-
-# %%
-if True:
-    out = cons.poll()

vision_v3/00_watch_for_new_videos/watch_and_fix_permissions_old.py

@@ -1,162 +0,0 @@
-import os
-import subprocess as sp
-import time
-import psutil
-import stat
-import sys
-import logging
-from watchdog.observers.polling import PollingObserver
-from watchdog.events import FileSystemEventHandler
-from threading import Thread
-from CommonCode import wq
-
-from CommonCode.settings import get_logger, LogColorize
-import argparse
-pfm = LogColorize.watch_and_fix_permissions
-if not ('__file__' in vars() or '__file__' in globals()):
-    __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
-
-parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-sys.path.append(parent_dir)
-
-from common import settings
-default_dir_watch = settings.dir_watch
-# %%
-logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
-
-logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
-
-try:
-    from CommonCode import kwq
-    publish = kwq.producer.send
-    topic = kwq.TOPICS.exit_00_videos_to_score_detection
-except Exception as e:
-    logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
-
-def decide_to_put_in_queue(file_name):
-    disqualifiers = list()
-    disqualifiers.append("_reduced" in file_name)
-    disqualifiers.append("_trimmed" in file_name)
-
-    disqualified = any(disqualifiers)
-
-    if file_name.endswith(".mp4") and not disqualified:
-        return True
-    else:
-        return False
-
-
-
-class logic_execute(Thread):
-    def __init__(self, event):
-        logging.debug(f"EVENT: {event}")
-        Thread.__init__(self)
-        self.event = event
-
-    def run(self):
-        event = self.event
-        if isinstance(event, str):
-            src_path = event
-        else:
-            src_path = event.src_path
-
-        filef = os.path.abspath(os.path.join(os.getcwd(), src_path))
-        logging.debug(f"GOT_FILE: {pfm(filef)}")
-
-        broke_out = False
-        for i in range(120):
-            files = list()
-            for proc in psutil.process_iter():
-                try:
-                    if "ftp" in proc.name():
-                        files.append(proc.open_files())
-                except Exception:
-                    pass
-                
-            f = list()
-            for x in files:
-                f.extend(x)
-
-            if any([filef == e.path for e in f]):
-                pass
-            else:
-                if decide_to_put_in_queue(filef):
-                    dirname = os.path.dirname(filef)
-                    foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
-                    if oct(foldmode) != "0o777":
-                        os.system(f"sudo chmod 777 {dirname}")
-
-                    logging.info(f"SETTING_PERMISSIONS: {dirname}")
-                    broke_out = True
-                    break
-
-            time.sleep(1)
-            
-        if not broke_out:
-            logging.info(f"TIMED_OUT: {filef}")
-
-            
-        chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
-        
-        try:
-            os.chmod(filef, 0o777)
-        except Exception as e:
-            logging.error(e)
-
-        if decide_to_put_in_queue(filef):
-            logging.info(f"QUEUE_PUT: {pfm(filef)}")
-            wq.publish(wq.TOPICS.ml_vision_to_score, filef)
-            try:
-                future = publish(topic, key=filef, value={"filepath": filef})
-            except Exception as e:
-                logging.error("KAFKA ERROR " + str(e))
-        else:
-            logging.info(f"QUEUE_DO_NOT_PUT: {filef}")
-
-
-class Handler(FileSystemEventHandler):
-    def on_created(self, event):
-        try:
-            self.handle_it(event)
-        except Exception as e:
-            logging.error(e)
-
-    def handle_it(self, event):
-        if event.is_directory:
-            return
-        lclass = logic_execute(event)
-        lclass.start()
-
-
-def observe_path(path):
-    event_handler = Handler()
-    observer = PollingObserver()
-    logger.info(f"Monitoring {path}")
-    observer.schedule(event_handler, path, recursive=True)
-    observer.start()
-    observer.is_alive()
-    return observer
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        prog="Watch and Fix Permissions And Push to Kafka Queue"
-    )
-
-    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
-
-    args, _ = parser.parse_known_args()
-    paths = args.paths
-    if len(paths) == 0:
-        paths = default_dir_watch
-    observers = [observe_path(path) for path in paths]
-    
-    try:
-        while True:
-            time.sleep(1)
-    except Exception as e:
-        logging.error(str(e))
-
-    for observer in observers:
-        observer.stop()
-        observer.join()

vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip.py

@@ -0,0 +1,150 @@
+from common_code.settings import get_logger, LogColorize
+import logging
+pfm = LogColorize.score_obj_det_embed
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip.py'
+
+
+logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
+
+import sys
+sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
+from cuda_objdet_clip import infer
+from common_code import kwq, util, file_names, vector_utils
+from common_code.video_meta import FTPVideo
+import os
+import sys
+import time
+import json
+import traceback
+
+
+client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
+
+# Call logger twice to reclaim
+logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip'+client_suffix, stdout=True, systemd=False, level = logging.INFO)
+logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
+
+
+pref = ''
+priority_queue = pref + kwq.TOPICS.enter_60_videos_embed_priority 
+regular_queue = pref + kwq.TOPICS.exit_00_videos_to_score_detection
+backfill_queue = pref + kwq.TOPICS.enter_60_videos_embed_backfill
+invalid_file_topic = kwq.TOPICS.exit_60_videos_invalid
+failed_embed_topic = kwq.TOPICS.exit_60_embedding_failed
+success_embed_topic = kwq.TOPICS.exit_60_videos_embedded
+
+
+group_id = "vision_v3"
+client_id = "objdet_and_embedder"+client_suffix
+
+
+
+queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_regular.subscribe([regular_queue])
+
+
+queue_priority = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_priority.subscribe([priority_queue])
+
+queue_backfill = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
+queue_backfill.subscribe([backfill_queue])
+
+
+
+logging.debug(f'KAFKA: Created subscribers to {regular_queue} and {priority_queue}')
+
+
+def check_file_validity(file_to_score):
+    if not os.path.exists(file_to_score):
+        return False, "file n/a"
+
+    if os.stat(file_to_score).st_size < 100*1000:  #If less than 100kb don't score it
+        return False, "file too small"
+
+    if '_reduced' in file_to_score:
+        return False, 'file reduced resolution'
+
+
+    if '_trimmed' in file_to_score:
+        return False, 'file is trimmed'
+
+    if not file_to_score.endswith('.mp4'):
+        return False, "Not mp4"
+
+    if len(FTPVideo(file_to_score).get_frames_info()) == 0:
+        return False, "Failed getting frame info"
+    
+    return True, ""
+
+
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+
+def perform_loop_kafka():
+    score_this = None
+    high_priority = False
+    result_priority = queue_priority.poll(timeout=0.05)
+    msg_value = None
+    if result_priority is not None:
+        score_this = get_score_file(result_priority)
+        msg_value = result_priority.value()
+        high_priority = True
+        logging.info(f"KAFKA: high priority file {pfm(score_this)}")
+        queue_priority.commit()
+    else:
+        result_regular = queue_regular.poll(timeout=0.05)
+        if result_regular is not None:
+            msg_value = result_regular.value()            
+            score_this = get_score_file(result_regular)
+            logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
+            queue_regular.commit()
+        else:
+            result_backfill = queue_backfill.poll(timeout=0.05)
+            if result_backfill is not None:
+                msg_value = result_backfill.value()            
+                score_this = get_score_file(result_backfill)
+                logging.info(f"KAFKA: backfill file {pfm(score_this)}")
+                queue_backfill.commit()
+            
+    
+
+
+    if score_this is None:
+        return
+
+    file_to_score = file_names.resolve_file_location(score_this)
+    if file_to_score is None:
+        return
+    if file_to_score != score_this:
+        logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
+        
+    file_is_valid, message = check_file_validity(file_to_score)
+
+    if not file_is_valid:
+        logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
+        kwq.publish(invalid_file_topic, file_to_score, message)
+    else:
+        logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
+        try:
+            det_scores, clip_scores = infer.score_video_cached(file_to_score, return_dict = True)
+            vector_utils.upload_vectors_to_db( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
+            kwq.publish(success_embed_topic, file_to_score) 
+            logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
+#            vector_utils.upload_to_doris( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
+            logging.info(f'UPLOADED TO DB: {pfm(file_to_score)}')
+        except Exception as e:
+            traceback.print_exc()
+            logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
+            kwq.publish(failed_embed_topic, file_to_score, str(e))
+
+
+
+while True:
+    perform_loop_kafka()
+
+
+
+

vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip_doris.py

@@ -0,0 +1,148 @@
+from common_code.settings import get_logger, LogColorize
+import logging
+pfm = LogColorize.score_obj_det_embed
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip.py'
+
+
+logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
+
+import sys
+sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
+from cuda_objdet_clip import infer
+from common_code import kwq, util, file_names, vector_utils
+from common_code.video_meta import FTPVideo
+import os
+import sys
+import time
+import json
+import traceback
+
+
+
+# Call logger twice to reclaim
+logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
+logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
+
+
+pref = ''
+priority_queue = pref + kwq.TOPICS.enter_60_videos_embed_priority 
+regular_queue = pref + kwq.TOPICS.exit_00_videos_to_score_detection
+backfill_queue = pref + kwq.TOPICS.enter_60_videos_embed_backfill
+invalid_file_topic = kwq.TOPICS.exit_60_videos_invalid
+failed_embed_topic = kwq.TOPICS.exit_60_embedding_failed
+success_embed_topic = kwq.TOPICS.exit_60_videos_embedded
+
+client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
+
+group_id = "vision_v3"
+client_id = "objdet_and_embedder"+client_suffix
+
+
+
+queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_regular.subscribe([regular_queue])
+
+
+queue_priority = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_priority.subscribe([priority_queue])
+
+queue_backfill = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
+queue_backfill.subscribe([backfill_queue])
+
+
+
+logging.debug(f'KAFKA: Created subscribers to {regular_queue} and {priority_queue}')
+
+
+def check_file_validity(file_to_score):
+    if not os.path.exists(file_to_score):
+        return False, "file n/a"
+
+    if os.stat(file_to_score).st_size < 100*1000:  #If less than 100kb don't score it
+        return False, "file too small"
+
+    if '_reduced' in file_to_score:
+        return False, 'file reduced resolution'
+
+
+    if '_trimmed' in file_to_score:
+        return False, 'file is trimmed'
+
+    if not file_to_score.endswith('.mp4'):
+        return False, "Not mp4"
+
+    if len(FTPVideo(file_to_score).get_frames_info()) == 0:
+        return False, "Failed getting frame info"
+    
+    return True, ""
+
+
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+
+def perform_loop_kafka():
+    score_this = None
+    high_priority = False
+    result_priority = queue_priority.poll(timeout=0.05)
+    msg_value = None
+    if result_priority is not None:
+        score_this = get_score_file(result_priority)
+        msg_value = result_priority.value()
+        high_priority = True
+        logging.info(f"KAFKA: high priority file {pfm(score_this)}")
+        queue_priority.commit()
+    else:
+        result_regular = queue_regular.poll(timeout=0.05)
+        if result_regular is not None:
+            msg_value = result_regular.value()            
+            score_this = get_score_file(result_regular)
+            logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
+            queue_regular.commit()
+        else:
+            result_backfill = queue_backfill.poll(timeout=0.05)
+            if result_backfill is not None:
+                msg_value = result_backfill.value()            
+                score_this = get_score_file(result_backfill)
+                logging.info(f"KAFKA: backfill file {pfm(score_this)}")
+                queue_backfill.commit()
+            
+    
+
+
+    if score_this is None:
+        return
+
+    file_to_score = file_names.resolve_file_location(score_this)
+    if file_to_score is None:
+        return
+    if file_to_score != score_this:
+        logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
+        
+    file_is_valid, message = check_file_validity(file_to_score)
+
+    if not file_is_valid:
+        logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
+        kwq.publish(invalid_file_topic, file_to_score, message)
+    else:
+        logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
+        try:
+            det_scores, clip_scores = infer.score_video_cached(file_to_score, return_dict = True)
+            vector_utils.upload_vectors_to_db( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
+            kwq.publish(success_embed_topic, file_to_score) 
+            logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
+        except Exception as e:
+            traceback.print_exc()
+            logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
+            kwq.publish(failed_embed_topic, file_to_score, str(e))
+
+
+
+while True:
+    perform_loop_kafka()
+
+
+
+

vision_v3/10_do_obj_det_and_clip/test_upload.py

@@ -0,0 +1,4 @@
+from common_code.vector_utils import upload_vectors_to_db
+
+
+upload_vectors_to_db('/srv/ftp_local/backyard/2026/07/03/backyard_00_20260703192858.mp4')

vision_v3/10_do_obj_det_and_clip/upload_npz_to_ch.py

@@ -0,0 +1,57 @@
+from common_code.vector_utils import upload_vectors_to_db
+from common_code import kwq, util, file_names, vector_utils
+from multiprocessing import Pool
+from tqdm import tqdm
+import argparse
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+
+
+topic_subscribe = 'upload_to_ch_db'
+def add_files_to_queue(paths, dry_run = False, force = False, do_upload = True, do_queue = False):
+    queued = set()
+    for rt in paths:
+        for root, dirs, files in os.walk(rt):
+            for f in files:
+                new_path = os.path.join(root, f)
+                if new_path.endswith('detection.npz') and 'leopards' not in new_path.lower():
+#                if decide_to_put_in_queue(new_path, force = force):
+                    queued.add(new_path)
+
+    for x in tqdm(queued):
+        try:
+            if do_upload:
+                upload_vectors_to_db(x)
+            if do_queue:
+                kwq.publish(topic_subscribe, x, x)
+
+        except Exception as e:
+            print('Failed: '+str(x)+'with '+str(e))
+
+        
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Watch and Fix Permissions And Push to Kafka Queue"
+    )
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+    parser.add_argument("--dry-run", action='store_true', help='Dry Run')
+    parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
+    parser.add_argument("--skip_upload", action='store_true', help='Force adding (do not check if already done)')            
+    parser.add_argument("--do_queue", action='store_true', help='Force adding (do not check if already done)')        
+    args, _ = parser.parse_known_args()
+    paths = args.paths
+    force = args.force
+    dry_run = args.dry_run
+    do_queue = args.do_queue
+    do_upload = not args.skip_upload
+    if len(paths) == 0:
+        paths = default_dir_watch
+
+    add_files_to_queue(paths, dry_run = dry_run, force=force, do_upload = do_upload, do_queue = do_queue)
+

vision_v3/10_do_obj_det_and_clip/upload_npz_to_doris.py

@@ -0,0 +1,43 @@
+from common_code.vector_utils import upload_to_doris
+from tqdm import tqdm
+import argparse
+import os
+
+def add_files_to_queue(paths, dry_run = False, force = False):
+    queued = set()
+    for rt in paths:
+        for root, dirs, files in os.walk(rt):
+            for f in files:
+                new_path = os.path.join(root, f)
+                if new_path.endswith('.oclip_embeds.npz'):
+#                if decide_to_put_in_queue(new_path, force = force):
+                    queued.add(new_path)
+
+    for x in tqdm(queued):
+        try:
+            upload_to_doris(x)
+        except Exception as e:
+            print('Failed: '+str(x)+'with '+str(e))
+
+        
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Watch and Fix Permissions And Push to Kafka Queue"
+    )
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+    parser.add_argument("--dry-run", action='store_true', help='Dry Run')
+    parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')    
+    args, _ = parser.parse_known_args()
+    paths = args.paths
+    force = args.force
+    dry_run = args.dry_run
+
+    if len(paths) == 0:
+        paths = default_dir_watch
+
+    add_files_to_queue(paths, dry_run = dry_run, force=force)
+

vision_v3/10_do_obj_det_and_clip/upload_npz_to_milvus.py

@@ -0,0 +1,62 @@
+from common_code.vector_utils import upload_vectors_to_db
+from common_code import kwq, util, file_names, vector_utils, settings
+from multiprocessing import Pool
+from tqdm import tqdm
+import argparse
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+
+
+topic_subscribe = 'upload_to_vector_db'
+def add_files_to_queue(paths, dry_run = False, force = False, do_upload = True, do_queue = False):
+    queued = set()
+    for rt in paths:
+        for root, dirs, files in os.walk(rt):
+            for f in files:
+                new_path = os.path.join(root, f)
+                if new_path.endswith('.oclip_embeds.npz'):
+#                if decide_to_put_in_queue(new_path, force = force):
+                    queued.add(new_path)
+
+    for x in tqdm(queued):
+        try:
+            if do_upload:
+                if os.path.exists(x+'.in_queue'):
+                    pass
+                else:
+                    upload_vectors_to_db(x)
+                    with open(x+'.in_queue','w') as ff:
+                        pass
+            if do_queue:
+                kwq.publish(topic_subscribe, x, x)
+
+        except Exception as e:
+            print('Failed: '+str(x)+'with '+str(e))
+
+        
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Watch and Fix Permissions And Push to Kafka Queue"
+    )
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+    parser.add_argument("--dry-run", action='store_true', help='Dry Run')
+    parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
+    parser.add_argument("--skip_upload", action='store_true', help='Force adding (do not check if already done)')            
+    parser.add_argument("--do_queue", action='store_true', help='Force adding (do not check if already done)')        
+    args, _ = parser.parse_known_args()
+    paths = args.paths
+    force = args.force
+    dry_run = args.dry_run
+    do_queue = args.do_queue
+    do_upload = not args.skip_upload
+    if len(paths) == 0:
+        paths = settings.dir_watch
+
+    add_files_to_queue(paths, dry_run = dry_run, force=force, do_upload = do_upload, do_queue = do_queue)
+

vision_v3/20_delete_files_and_create_other_videos/20_add_into_queue.py

@@ -0,0 +1,42 @@
+
+
+import os
+import sys
+import time
+from common_code import kwq
+from common_code.util import get_cset_match
+from common_code.file_names import resolve_file_location
+import json
+import sys
+import logging
+from common_code.settings import get_logger, LogColorize
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/model_runner/")
+from video_reduce_resolution import convert_file_with_check
+from common_code import settings
+default_dir_watch = settings.dir_watch
+
+
+pfm = LogColorize.file_modifications
+logger = get_logger(__name__,'/var/log/ml_vision_logs/20_add_into_queue', stdout=True, systemd=False, level = logging.INFO)
+
+success_det_and_embed_topic = kwq.TOPICS.exit_60_videos_embedded
+
+
+
+f_nav = list()
+for c_dir in settings.dir_watch_archived():
+    for root, _, files in  os.walk(c_dir):
+        for f in files:
+            if f.endswith('.mp4') and "reduced" not in f:
+                cfile = os.path.join( root, f)
+                f_nav.append(cfile)
+
+
+# %%
+#for f, _ in zip(f_nav, range(10)):
+from tqdm import tqdm
+
+    kwq.publish(success_det_and_embed_topic, f)
+
+# %%
+

vision_v3/20_delete_files_and_create_other_videos/20_delete_files_and_create_other_videos.py

@@ -0,0 +1,154 @@
+
+import os
+import sys
+import time
+from common_code import kwq
+from common_code.util import get_cset_match
+from common_code.file_names import resolve_file_location
+import json
+import sys
+import logging
+from common_code.settings import get_logger, LogColorize
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/model_runner/")
+from video_reduce_resolution import convert_file_with_check
+import numpy as np
+
+pfm = LogColorize.file_modifications
+logger = get_logger(__name__,'/var/log/ml_vision_logs/20_delete_files_and_create_other_videos', stdout=True, systemd=False, level = logging.INFO)
+
+success_det_and_embed_topic = kwq.TOPICS.exit_60_videos_embedded
+
+group_id = "vision_v3"
+client_id = "file_converter"
+
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/20_delete_files_and_create_other_videos/20_delete_files_and_create_other_videos.py'
+
+
+logger.info(pfm(f"Starting file modifier @ {__file__}"))
+
+
+queue = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
+queue.subscribe([success_det_and_embed_topic])
+
+# %%
+def get_ok_to_delete(file_path):
+    max_look = 5
+    c_path = file_path
+    settings_default = {'class_threshold':10000, 'frames_with_dets_threshold': -10000, 'species_ignore': [], 'keep_reduced_mp4': False}
+
+    depth_set = dict()
+    for i in range(max_look):
+        c_path  = os.path.abspath(os.path.join(c_path, '..'))
+        test_path = os.path.join(c_path, 'settings.json')
+        if os.path.exists(test_path):
+            with open(test_path, 'r') as rr:
+                depth_set[i] = json.load(rr)
+
+    depth_set = dict(sorted(depth_set.items(), reverse=True))
+
+    for depth, val in depth_set.items():
+        settings_default.update(val)
+
+
+    settings_default['species_ignore'] = set(settings_default['species_ignore'])
+    return settings_default
+# %%
+
+def exec_file_remove_logic(cset, dry_run = False):
+
+    
+    did_purge = False
+    if isinstance(cset, dict):
+        file_path = cset['.mp4']
+    else:
+        file_path = cset
+        cset = get_cset_match(file_path)
+    
+    settings = get_ok_to_delete(file_path)
+
+    if settings['class_threshold'] > 1 or settings['class_threshold'] <= 0:
+        print(f"THRESHOLD SET TO ABOVE 1, SKIPPING: {file_path}")
+        return False
+
+
+    if '.detection.npz' in cset and os.path.exists(cset['.detection.npz']):
+        ff = np.load(cset['.detection.npz'], allow_pickle = True)
+        det_results = dict(ff)
+
+        del_idx = list()
+        for idx, val in det_results['species_label_map'].item().items():
+            if val in settings['species_ignore']:
+                del_idx.append(idx)
+
+        keep_mask = ~np.isin(det_results['final_labels'], del_idx)
+
+        filt_scores = det_results['final_scores'][keep_mask]
+        filt_frames = det_results['final_frames'][keep_mask]
+
+        passes = filt_scores > settings['class_threshold']
+
+        n_frames_above = len(np.unique(filt_frames[passes]))
+        n_frames_scored = len(det_results['scored_frames'])
+
+        frac_frames_scored = n_frames_above / n_frames_scored
+        do_keep = frac_frames_scored > settings['frames_with_dets_threshold']
+
+
+        keep_reduced_mp4 = settings['keep_reduced_mp4']
+        if do_keep:
+            cpath = os.path.splitext(cset['.mp4'])[0] + '.has_objs'
+            with open(cpath,'w') as cc:
+                pass
+                did_purge = False
+        else:
+            did_purge = True
+
+            for ff, cf in cset.items():
+                if cf.endswith('npz') or cf.endswith('jpg'):
+                    pass
+                else:
+                    if not dry_run and os.path.exists(cf):
+                        if '_reduced' in cf:
+                            logger.info(f"Skipping reduced_mp4 deletion {pfm(cf)}")
+                            continue
+                            
+                        logger.info(f"Removing {pfm(cf)}")
+                    else:
+                        print(f'Skipping deleting {ff} because of dry run or path not exist')
+    return did_purge
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+from common_code.video_meta import FTPVideo
+
+
+def perform_kafka_loop():
+    try:
+#    if True:
+        result = queue.poll(timeout=0.05)
+        if result is None:
+            return
+        
+        queue.commit()
+        file_to_score = resolve_file_location(get_score_file(result))
+        if file_to_score is None:
+            logger.info(f"Not found {pfm(file_to_score)}")
+            return
+        logger.info(f"Executing file operations {pfm(file_to_score)}")
+        did_purge = exec_file_remove_logic(get_cset_match(file_to_score))
+        if not did_purge:
+            did_convert = convert_file_with_check(file_to_score)
+            FTPVideo(file_to_score).frames_info;
+        
+        logger.info(f"Finished file operations {pfm(file_to_score)}")        
+    except Exception as e:
+        logger.error(f"Failed: {pfm(file_to_score)}, {str(e)}") 
+
+
+while True:
+    perform_kafka_loop()
+
+
+

vision_v3/20_delete_files_and_create_other_videos/create_other_videos.py

@@ -0,0 +1,38 @@
+from CommonCode import wq
+from CommonCode.video_meta import FTPVideo
+import os
+import sys
+import time
+
+
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/scoring_scheduler")
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/model_runner/")
+
+from trim_with_birds import execute_file_trimming
+from video_reduce_resolution import convert_file_with_check
+input_topic = wq.TOPICS.ml_vision_objdet_results_purge_skipped
+output_if_success = wq.TOPICS.ml_vision_videos_modify_success
+output_if_fail = wq.TOPICS.ml_vision_videos_modify_failed
+
+
+
+def perform_loop():
+    file_to_score = wq.consume(input_topic)
+    if file_to_score is None:
+        return
+
+    try:
+        did_trim = False#        did_trim = execute_file_trimming(files_to_check = file_to_score)
+        did_convert = convert_file_with_check(file_to_score)
+        FTPVideo(file_to_score).frames_info;
+        wq.publish(output_if_success, file_to_score)
+        print(f"File trimmed:{did_trim}, did_downsize:{did_convert}, {file_to_score}")
+
+    except Exception as e:
+        wq.publish(output_if_fail, {file_to_score: str(e)})
+        print(f"File fail to create {file_to_score}, {e}")
+
+# %%
+while True:
+    perform_loop()
+    time.sleep(1)

vision_v3/20_delete_files_and_create_other_videos/insert_in_db.py

@@ -0,0 +1,10 @@
+
+det_npz = '/srv/ftp/railing/2025/11/17/railing_00_20251117140039.detection.npz'
+mp4_path = '/srv/ftp/railing/2025/11/17/railing_00_20251117140039.mp4'
+import numpy as np
+ff = np.load(det_npz, allow_pickle = True)
+det_results = dict(ff)
+
+del_idx = list()
+from common_code.video_meta import FTPVideo
+FTPVideo(mp4_path)

vision_v3/20_delete_files_and_create_other_videos/purge_file_if_no_dets.py

@@ -0,0 +1,52 @@
+import os
+import sys
+import time
+
+
+from CommonCode import wq, kwq
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/scoring_scheduler")
+sys.path.insert(0, "/home/thebears/Source/ml/sources_mltraining/model_runner/")
+from purge_without_birds import execute_file_purging
+
+from video_reduce_resolution import convert_file
+
+input_topic = wq.TOPICS.ml_vision_objdet_results_pg_success
+output_if_success = wq.TOPICS.ml_vision_objdet_results_purge_success
+output_if_fail = wq.TOPICS.ml_vision_objdet_results_purge_failed
+output_if_skipped = wq.TOPICS.ml_vision_objdet_results_purge_skipped
+
+
+topic_if_has_nuggets = kwq.TOPICS.exit_40_videos_with_nuggets
+topic_if_no_nuggets = kwq.TOPICS.exit_40_videos_without_nuggets
+
+
+
+def perform_loop():
+    file_to_score = wq.consume(input_topic)
+    if file_to_score is None:
+        return
+
+    try:
+        purged_result = execute_file_purging(files_to_check=file_to_score)
+        did_purge = False
+        if purged_result is not None and len(purged_result) > 0:
+            did_purge = purged_result[0]
+            
+        if did_purge:
+            wq.publish(output_if_success, file_to_score)
+            print(f"File purged {file_to_score}")
+            kwq.publish(topic_if_no_nuggets, file_to_score)
+        else:
+            wq.publish(output_if_skipped, file_to_score)
+            print(f"File not purged {file_to_score}")
+            kwq.publish(topic_if_has_nuggets, file_to_score)
+
+    except Exception as e:
+        wq.publish(output_if_fail, {file_to_score: str(e)})
+        print(f"File failed to purge {file_to_score}, {e}")
+
+
+# %%
+while True:
+    perform_loop()
+    time.sleep(1)

vision_v3/20_delete_files_and_create_other_videos/upload_from_folder.py

@@ -0,0 +1,75 @@
+from pymilvus import MilvusClient, DataType
+import numpy as np
+import time
+from pymilvus.client.types import LoadState
+client = MilvusClient(
+    uri="http://localhost:19530"
+)
+
+
+res = client.get_load_state(
+    collection_name="nuggets_so400m"
+)
+if res['state'] == LoadState.Loaded:
+    pass
+else:
+    client.load_collection(collection_name = 'nuggets_so400m')
+    for i in range(10):
+        time.sleep(1)
+        if res['state'] == LoadState.Loaded:
+            break
+
+
+def get_vec_path(vpath):
+    return os.path.splitext(vpath)[0]+'.oclip_embeds.npz'
+
+def get_db_embed_done_path(vpath):
+    return os.path.splitext(vpath)[0]+'.db_has_oclip_embeds'
+
+
+def upload_vector_file(vector_file_to_upload):
+    if os.path.exists(get_embed_done_path(vector_file_to_upload)):
+        print('Already exists in DB, skipping upload')
+        return
+
+    vector_file_to_upload = get_vec_path(vector_file_to_upload)
+    vf = np.load(vector_file_to_upload)
+
+    embeds = vf['embeds']
+    fr_nums = vf['frame_numbers']
+
+    fname_root = vector_file_to_upload.rsplit('/',1)[-1].split('.')[0]
+    fc = fname_root.split('_')[-1]
+
+    data = list()
+    filepath = vector_file_to_upload.replace('/srv/ftp/','').replace('/mergedfs/ftp','').split('.')[-0]
+
+    for embed, frame_num in zip(embeds, fr_nums):
+        fg = '{0:05g}'.format(frame_num)
+        id_num = int(fc+fg)
+        to_put = dict(primary_id= id_num, filepath=filepath, frame_number = int(frame_num), so400m=embed)
+        data.append(to_put)
+
+    client.insert(collection_name = 'nuggets_so400m', data = data)
+    print(f'Inserting into DB, {vector_file_to_upload}')
+
+    with open(get_embed_done_path(vector_file_to_upload),'w') as ff:
+        ff.write(str(time.time()))
+
+
+
+
+
+root_path = '/srv/ftp/railing/2026/'
+to_put = list()
+for root, dirs, files in os.walk(root_path):
+    for x in files:
+        if x.endswith('oclip_embeds.npz'):
+            to_put.append(os.path.join(root, x))
+
+
+for x in to_put:
+    upload_vector_file(x)
+
+
+    

vision_v3/30_queued_upload_to_db/30_queued_upload_to_ch.py

@@ -0,0 +1,202 @@
+
+import numpy as np
+import time
+from multiprocessing import Process, Queue, Manager
+import threading
+from common_code import kwq, util, file_names, vector_utils, video_meta
+
+from datetime import datetime
+from clickhouse_connect import get_client
+
+
+topic_subscribe = 'upload_to_ch_db'
+group_id = "vision_v3_uploaders"
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+
+
+def insert_into_ch(file_path, client):
+    cset = util.get_cset_match(file_path)
+    ff = np.load(cset['.detection.npz'], allow_pickle = True)
+    det_results = dict(ff)
+    del_idx = list()
+
+    filt_scores = det_results['final_scores']
+    filt_frames = det_results['final_frames']
+    spec_label_idx = ff['species_label_map'].tolist()
+
+    ftp_vid = video_meta.FTPVideo(cset)
+
+    frame_num_off = ftp_vid.sorted_frames_info
+    frame_period = ftp_vid.frameperiod
+    frame_rate = ftp_vid.framerate
+
+    indexes = np.searchsorted(ff['scored_frames'], filt_frames)
+    dff = np.diff(ff['scored_frames'])
+
+    head = np.insert(dff, 0, 0)
+    tail = np.append(dff, 0)
+    all_frame_durations = frame_period*np.mean(np.stack([head,tail]), axis=0)
+
+
+    frame_file = [ftp_vid.cpath.split('/')[-1].replace('_reduced','').replace('.mp4','')]*len(filt_frames)
+    frame_bboxes = ff['final_boxes']
+    frame_numbers = filt_frames
+    frame_scores = filt_scores
+    frame_species = [spec_label_idx[int(x)] for x in ff['final_labels']]
+    frame_durations = all_frame_durations[indexes]
+    frame_tstamps = [frame_num_off[x]['time'] for x in filt_frames]
+    db_insert_times = [datetime.now()]*len(filt_frames)
+
+    a = frame_numbers
+    starts = np.r_[0, np.flatnonzero(a[1:] != a[:-1]) + 1]
+    det_count_num = np.arange(len(a)) - np.repeat(starts, np.diff(np.r_[starts, len(a)]))
+    camera =  [ftp_vid.camera_name] * len(filt_frames)
+
+
+    result = client.insert('nuggets.cameras_object_detection_results',
+                  column_oriented = True,
+                  column_names = ['camera','video_filename','frame_number','detection_number','classification_score','object_name','bbox_xyxy','frame_duration','frame_timestamp','db_insert_time'],
+                  data = [camera, frame_file, frame_numbers, det_count_num, frame_scores, frame_species, frame_bboxes.tolist(), frame_durations, frame_tstamps, db_insert_times]
+                  )
+
+
+# %%
+
+
+def watch_and_upload(rank, health_queue):
+    client = get_client(host='192.168.1.242', compression=True)
+    start_time = time.time()
+    client_suffix = '_'+str(rank)
+    client_id = 'uploader_to_ch_db'+client_suffix
+    current_time = time.time()    
+    health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
+    health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
+    
+    queue = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
+    queue.subscribe([topic_subscribe])
+
+    last_heartbeat = time.time()
+    
+    while True:
+        current_time = time.time()
+        
+        # # Exit after 5 minutes
+        # if (current_time - start_time) > 60*5:
+        #     print('###########################')
+        #     print('######### EXITING #########')
+        #     print('###########################')            
+        #     return
+
+        # Send heartbeat every 10 seconds
+        if current_time - last_heartbeat > 5:
+            try:
+                health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
+                last_heartbeat = current_time
+            except:
+                # Queue might be full, continue working
+                pass
+        
+        result = queue.poll(timeout=0.05)    
+        if result is not None:
+            try:
+                score_this = get_score_file(result)                
+                insert_into_ch(score_this, client)
+                print('#' + str(rank) + ' Uploaded to ' + score_this)
+                # Send success heartbeat
+                try:
+                    health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'working'})
+                except:
+                    pass
+            except Exception as e:
+                print('Failed ' + score_this + ':' + str(e))
+                
+            queue.commit()
+
+def health_monitor(processes, health_queues, num_workers):
+    """Monitor process health and restart hung processes"""
+    last_heartbeat = {}
+    HEALTH_TIMEOUT = 120  # Consider process hung if no heartbeat for 60 seconds
+    
+    while True:
+        time.sleep(5)  # Check every 5 seconds
+        current_time = time.time()
+        
+        for rank in range(num_workers):
+            # Drain the health queue for this process
+            latest_heartbeat = None
+            try:
+                while True:
+                    try:
+                        heartbeat = health_queues[rank].get_nowait()
+                        if heartbeat['rank'] == rank:
+                            latest_heartbeat = heartbeat['timestamp']
+                    except:
+                        break
+            except:
+                pass
+            
+            # Update last known heartbeat
+            if latest_heartbeat:
+                last_heartbeat[rank] = latest_heartbeat
+            
+            # Check if process is alive and responsive
+            process = processes[rank]
+            last_seen = last_heartbeat.get(rank, 0)
+            
+            if not process.is_alive():
+                print(f"Process {rank} has died. Restarting...")
+                processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+                processes[rank].start()
+                last_heartbeat[rank] = current_time
+            elif current_time - last_seen > HEALTH_TIMEOUT:
+                print(f"Process {rank} appears hung (no heartbeat for {current_time - last_seen:.1f}s). Restarting...")
+                process.terminate()
+                process.join(timeout=5)
+                if process.is_alive():
+                    process.kill()
+                    process.join()
+                
+                # Create new process
+                processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+                processes[rank].start()
+                last_heartbeat[rank] = current_time
+
+# Main execution
+if __name__ == "__main__":
+    num_workers = 4
+    
+    # Create health queues for each worker
+    health_queues = [Queue(maxsize=10) for _ in range(num_workers)]
+    
+    # Create and start worker processes
+    processes = []
+    for rank in range(num_workers):
+        p = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+        p.start()
+        processes.append(p)
+    
+    # Start health monitor in a separate thread
+    monitor_thread = threading.Thread(
+        target=health_monitor, 
+        args=(processes, health_queues, num_workers),
+        daemon=True
+    )
+    monitor_thread.start()
+    
+    try:
+        # Wait for all processes to complete (they will exit after 5 minutes)
+        for p in processes:
+            p.join()
+        print("All worker processes completed")
+    except KeyboardInterrupt:
+        print("Shutting down...")
+        for p in processes:
+            p.terminate()
+            p.join(timeout=5)
+            if p.is_alive():
+                p.kill()
+                p.join()
+

vision_v3/30_queued_upload_to_db/30_queued_upload_to_db.py

@@ -0,0 +1,142 @@
+from common_code.vector_utils import upload_vectors_to_db
+from common_code import kwq, util, file_names, vector_utils
+import time
+from multiprocessing import Process, Queue, Manager
+import threading
+
+topic_subscribe = 'upload_to_vector_db'
+group_id = "vision_v3_uploaders"
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+def watch_and_upload(rank, health_queue):
+    start_time = time.time()
+    client_suffix = '_'+str(rank)
+    client_id = 'uploader_to_vector_db'+client_suffix
+
+    queue = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
+    queue.subscribe([topic_subscribe])
+
+    last_heartbeat = time.time()
+    
+    while True:
+        current_time = time.time()
+        
+        # # Exit after 5 minutes
+        # if (current_time - start_time) > 60*5:
+        #     print('###########################')
+        #     print('######### EXITING #########')
+        #     print('###########################')            
+        #     return
+
+        # Send heartbeat every 10 seconds
+        if current_time - last_heartbeat > 5:
+            try:
+                health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
+                last_heartbeat = current_time
+            except:
+                # Queue might be full, continue working
+                pass
+        
+        result = queue.poll(timeout=0.05)    
+        if result is not None:
+            score_this = get_score_file(result)
+            try:
+                upload_vectors_to_db(score_this)
+                print('#' + str(rank) + ' Uploaded to ' + score_this)
+                # Send success heartbeat
+                try:
+                    health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'working'})
+                except:
+                    pass
+            except Exception as e:
+                print('Failed ' + score_this + ':' + str(e))
+                
+            queue.commit()
+
+def health_monitor(processes, health_queues, num_workers):
+    """Monitor process health and restart hung processes"""
+    last_heartbeat = {}
+    HEALTH_TIMEOUT = 15  # Consider process hung if no heartbeat for 60 seconds
+    
+    while True:
+        time.sleep(5)  # Check every 5 seconds
+        current_time = time.time()
+        
+        for rank in range(num_workers):
+            # Drain the health queue for this process
+            latest_heartbeat = None
+            try:
+                while True:
+                    try:
+                        heartbeat = health_queues[rank].get_nowait()
+                        if heartbeat['rank'] == rank:
+                            latest_heartbeat = heartbeat['timestamp']
+                    except:
+                        break
+            except:
+                pass
+            
+            # Update last known heartbeat
+            if latest_heartbeat:
+                last_heartbeat[rank] = latest_heartbeat
+            
+            # Check if process is alive and responsive
+            process = processes[rank]
+            last_seen = last_heartbeat.get(rank, 0)
+            
+            if not process.is_alive():
+                print(f"Process {rank} has died. Restarting...")
+                processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+                processes[rank].start()
+                last_heartbeat[rank] = current_time
+            elif current_time - last_seen > HEALTH_TIMEOUT:
+                print(f"Process {rank} appears hung (no heartbeat for {current_time - last_seen:.1f}s). Restarting...")
+                process.terminate()
+                process.join(timeout=5)
+                if process.is_alive():
+                    process.kill()
+                    process.join()
+                
+                # Create new process
+                processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+                processes[rank].start()
+                last_heartbeat[rank] = current_time
+
+# Main execution
+if __name__ == "__main__":
+    num_workers = 4
+    
+    # Create health queues for each worker
+    health_queues = [Queue(maxsize=10) for _ in range(num_workers)]
+    
+    # Create and start worker processes
+    processes = []
+    for rank in range(num_workers):
+        p = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
+        p.start()
+        processes.append(p)
+    
+    # Start health monitor in a separate thread
+    monitor_thread = threading.Thread(
+        target=health_monitor, 
+        args=(processes, health_queues, num_workers),
+        daemon=True
+    )
+    monitor_thread.start()
+    
+    try:
+        # Wait for all processes to complete (they will exit after 5 minutes)
+        for p in processes:
+            p.join()
+        print("All worker processes completed")
+    except KeyboardInterrupt:
+        print("Shutting down...")
+        for p in processes:
+            p.terminate()
+            p.join(timeout=5)
+            if p.is_alive():
+                p.kill()
+                p.join()
+

vision_v3/40_do_cropped_clip/40_do_cropped_clip.py

@@ -0,0 +1,92 @@
+from common_code.settings import get_logger, LogColorize
+import logging
+pfm = LogColorize.score_obj_det_embed
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/40_do_cropped_clip/40_do_cropped_clip.py'
+
+
+log_file = '/var/log/ml_vision_logs/40_do_cropped_clip'
+logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
+
+import sys
+sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
+from cuda_objdet_clip import infer
+from common_code import kwq, util, file_names, vector_utils
+from common_code.video_meta import FTPVideo
+import os
+import sys
+import time
+import json
+import traceback
+
+
+client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
+
+# Call logger twice to reclaim
+logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
+logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
+
+
+pref = ''
+regular_queue = pref + kwq.TOPICS.exit_60_videos_embedded
+
+
+
+group_id = "vision_v3"
+client_id = "cropped_embedder"+client_suffix
+
+
+queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_regular.subscribe([regular_queue])
+
+logging.debug(f'KAFKA: Created subscribers to {regular_queue}')
+
+def check_file_validity(file_to_score):
+    if 'leopard' in file_to_score.lower():
+        return True, ''
+    else:
+        return False, ''
+    
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+def perform_loop_kafka():
+    score_this = None
+    msg_value = None
+    result_regular = queue_regular.poll(timeout=0.05)
+    if result_regular is not None:
+        msg_value = result_regular.value()            
+        score_this = get_score_file(result_regular)
+        logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
+        queue_regular.commit()
+    
+    if score_this is None:
+        return
+
+    file_to_score = file_names.resolve_file_location(score_this)
+    if file_to_score is None:
+        return
+
+    
+    
+    
+    if file_to_score != score_this:
+        logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
+
+    file_is_valid, message = check_file_validity(file_to_score)
+    if not file_is_valid:
+        logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
+        return
+
+    logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
+    try:
+        infer.score_video_sub_clip_cached( file_to_score)
+        logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
+    except Exception as e:
+        traceback.print_exc()
+        logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
+
+while True:
+    perform_loop_kafka()
+