YACWC
This commit is contained in:
+116
@@ -0,0 +1,116 @@
|
||||
#!/home/thebears/envs/ml_vision_objdet/bin/python
|
||||
import os
|
||||
|
||||
import subprocess as sp
|
||||
import time
|
||||
import psutil
|
||||
import stat
|
||||
import sys
|
||||
import logging
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from threading import Thread
|
||||
from tqdm import tqdm
|
||||
from common_code import kwq
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import argparse
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
|
||||
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.append(parent_dir)
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
|
||||
from common_code import settings
|
||||
|
||||
#default_dir_watch = settings.dir_watch_missed
|
||||
default_dir_watch = settings.dir_watch
|
||||
|
||||
publish = kwq.producer.send
|
||||
#topic = kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
topic = kwq.TOPICS.enter_60_videos_embed_backfill
|
||||
|
||||
necessary_extensions = [
|
||||
'.oclip_embeds.npz',
|
||||
'.detection.npz']
|
||||
|
||||
def decide_to_put_in_queue(file_name, force = False):
|
||||
if os.path.getsize(file_name) < 1*1000*1000:
|
||||
return False
|
||||
disqualifiers = list()
|
||||
disqualifiers.append(not os.path.exists(file_name))
|
||||
disqualifiers.append("_reduced" in file_name)
|
||||
disqualifiers.append("_trimmed" in file_name)
|
||||
# disqualifiers.append('hummingbird' in file_name)
|
||||
|
||||
disqualified = any(disqualifiers)
|
||||
if file_name.endswith(".mp4") and not disqualified:
|
||||
rt_path, _ = os.path.splitext(file_name)
|
||||
if force:
|
||||
all_exist = False
|
||||
else:
|
||||
all_exist = all([os.path.exists( rt_path + ext) for ext in necessary_extensions])
|
||||
return (not all_exist)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.append(parent_dir)
|
||||
def place_file_in_queue(filef):
|
||||
dirname = os.path.dirname(filef)
|
||||
foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
|
||||
if oct(foldmode) != "0o777":
|
||||
os.system(f"sudo chmod 777 {dirname}")
|
||||
|
||||
logging.info(f"SETTING_PERMISSIONS: {dirname}")
|
||||
|
||||
chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
|
||||
try:
|
||||
|
||||
os.chmod(filef, 0o777)
|
||||
except Exception as e:
|
||||
pass
|
||||
#$ logging.error(e)
|
||||
|
||||
logging.info(f"QUEUE_PUT: {filef}")
|
||||
publish(topic, key=filef, value={"filepath": filef})
|
||||
|
||||
|
||||
|
||||
def add_files_to_queue(paths, dry_run = False, force = False):
|
||||
queued = set()
|
||||
for rt in paths:
|
||||
for root, dirs, files in os.walk(rt):
|
||||
for f in files:
|
||||
new_path = os.path.join(root, f)
|
||||
if decide_to_put_in_queue(new_path, force = force):
|
||||
queued.add(new_path)
|
||||
|
||||
|
||||
for x in tqdm(queued):
|
||||
if dry_run:
|
||||
print('DRY RUN, CANDIDATES FOR QUEUE: ' + x)
|
||||
else:
|
||||
place_file_in_queue(x)
|
||||
|
||||
|
||||
# %%
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Watch and Fix Permissions And Push to Kafka Queue"
|
||||
)
|
||||
|
||||
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
|
||||
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
|
||||
parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
|
||||
args, _ = parser.parse_known_args()
|
||||
paths = args.paths
|
||||
force = args.force
|
||||
dry_run = args.dry_run
|
||||
|
||||
if len(paths) == 0:
|
||||
paths = default_dir_watch
|
||||
|
||||
add_files_to_queue(paths, dry_run = dry_run, force=force)
|
||||
|
||||
|
||||
+12
-14
@@ -8,36 +8,34 @@ import logging
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from threading import Thread
|
||||
from CommonCode import wq
|
||||
from CommonCode.util import is_ipython
|
||||
from common_code.util import is_ipython
|
||||
import queue
|
||||
from CommonCode.settings import get_logger, LogColorize
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import argparse
|
||||
|
||||
from common_code import kwq
|
||||
import subprocess
|
||||
from collections import deque
|
||||
|
||||
|
||||
publish = kwq.producer.send
|
||||
topic = kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
|
||||
|
||||
pfm = LogColorize.watch_and_fix_permissions
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
|
||||
__file__ = '/home/thebears/Source/pipelines/vision_v3/00_watch_for_new_videos/00_watch_and_fix_permissions.py'
|
||||
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.append(parent_dir)
|
||||
|
||||
from common import settings
|
||||
from common_code import settings
|
||||
default_dir_watch = settings.dir_watch
|
||||
|
||||
tasks = queue.Queue()
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
|
||||
|
||||
logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
|
||||
logger.info(pfm(f"Starting file watcher @ {__file__}"))
|
||||
|
||||
try:
|
||||
from CommonCode import kwq
|
||||
publish = kwq.producer.send
|
||||
topic = kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
except Exception as e:
|
||||
logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
|
||||
|
||||
def decide_to_put_in_queue(file_name):
|
||||
disqualifiers = list()
|
||||
@@ -112,7 +110,7 @@ def process_file(filef):
|
||||
|
||||
if decide_to_put_in_queue(filef):
|
||||
logging.info(f"QUEUE_PUT: {pfm(filef)}")
|
||||
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
|
||||
|
||||
try:
|
||||
future = publish(topic, key=filef, value={"filepath": filef})
|
||||
except Exception as e:
|
||||
Binary file not shown.
@@ -0,0 +1,20 @@
|
||||
from load_missed_lib import place_file_in_queue
|
||||
|
||||
# %%
|
||||
import os
|
||||
|
||||
rrt = ['/srv/ftp_tcc/', '/mnt/hdd_24tb_1/videos/ftp']
|
||||
dir_scan = ['leopards3','leopards4']
|
||||
|
||||
|
||||
rt_scans = list()
|
||||
|
||||
for r1 in rrt:
|
||||
for d1 in dir_scan:
|
||||
rt_scans.append(os.path.join(r1, d1))
|
||||
|
||||
# %%
|
||||
for rt_s in rt_scans:
|
||||
for root, dirs, files in os.walk(rt_s):
|
||||
for x in files:
|
||||
print(x)
|
||||
+6
-7
@@ -10,9 +10,9 @@ import logging
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from threading import Thread
|
||||
from CommonCode import wq
|
||||
from CommonCode import kwq
|
||||
from CommonCode.settings import get_logger, LogColorize
|
||||
|
||||
from common_code import kwq
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import argparse
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
|
||||
@@ -20,7 +20,7 @@ if not ('__file__' in vars() or '__file__' in globals()):
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.append(parent_dir)
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
|
||||
from common import settings
|
||||
from common_code import settings
|
||||
|
||||
#default_dir_watch = settings.dir_watch_missed
|
||||
default_dir_watch = settings.dir_watch
|
||||
@@ -30,8 +30,7 @@ topic = kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
|
||||
necessary_extensions = [
|
||||
'.oclip_embeds.npz',
|
||||
'.json',
|
||||
'.db_has_sl2_embeds_part' ]
|
||||
'.detection.npz']
|
||||
|
||||
def decide_to_put_in_queue(file_name):
|
||||
if os.path.getsize(file_name) < 1*1000*1000:
|
||||
@@ -71,7 +70,7 @@ def place_file_in_queue(filef):
|
||||
|
||||
logging.info(f"QUEUE_PUT: {filef}")
|
||||
publish(topic, key=filef, value={"filepath": filef})
|
||||
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
|
||||
|
||||
|
||||
|
||||
def add_files_to_queue(paths, dry_run = False):
|
||||
@@ -1,11 +0,0 @@
|
||||
from CommonCode import kwq
|
||||
publish = kwq.producer.send
|
||||
topic = kwq.TOPICS.videos_to_score_detection
|
||||
cons = kwq.create_consumer()
|
||||
|
||||
# %%
|
||||
cons.subscribe([topic])
|
||||
|
||||
# %%
|
||||
if True:
|
||||
out = cons.poll()
|
||||
@@ -1,162 +0,0 @@
|
||||
import os
|
||||
import subprocess as sp
|
||||
import time
|
||||
import psutil
|
||||
import stat
|
||||
import sys
|
||||
import logging
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from threading import Thread
|
||||
from CommonCode import wq
|
||||
|
||||
from CommonCode.settings import get_logger, LogColorize
|
||||
import argparse
|
||||
pfm = LogColorize.watch_and_fix_permissions
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
|
||||
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.append(parent_dir)
|
||||
|
||||
from common import settings
|
||||
default_dir_watch = settings.dir_watch
|
||||
# %%
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
|
||||
|
||||
logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
|
||||
|
||||
try:
|
||||
from CommonCode import kwq
|
||||
publish = kwq.producer.send
|
||||
topic = kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
except Exception as e:
|
||||
logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
|
||||
|
||||
def decide_to_put_in_queue(file_name):
|
||||
disqualifiers = list()
|
||||
disqualifiers.append("_reduced" in file_name)
|
||||
disqualifiers.append("_trimmed" in file_name)
|
||||
|
||||
disqualified = any(disqualifiers)
|
||||
|
||||
if file_name.endswith(".mp4") and not disqualified:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
|
||||
class logic_execute(Thread):
|
||||
def __init__(self, event):
|
||||
logging.debug(f"EVENT: {event}")
|
||||
Thread.__init__(self)
|
||||
self.event = event
|
||||
|
||||
def run(self):
|
||||
event = self.event
|
||||
if isinstance(event, str):
|
||||
src_path = event
|
||||
else:
|
||||
src_path = event.src_path
|
||||
|
||||
filef = os.path.abspath(os.path.join(os.getcwd(), src_path))
|
||||
logging.debug(f"GOT_FILE: {pfm(filef)}")
|
||||
|
||||
broke_out = False
|
||||
for i in range(120):
|
||||
files = list()
|
||||
for proc in psutil.process_iter():
|
||||
try:
|
||||
if "ftp" in proc.name():
|
||||
files.append(proc.open_files())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
f = list()
|
||||
for x in files:
|
||||
f.extend(x)
|
||||
|
||||
if any([filef == e.path for e in f]):
|
||||
pass
|
||||
else:
|
||||
if decide_to_put_in_queue(filef):
|
||||
dirname = os.path.dirname(filef)
|
||||
foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
|
||||
if oct(foldmode) != "0o777":
|
||||
os.system(f"sudo chmod 777 {dirname}")
|
||||
|
||||
logging.info(f"SETTING_PERMISSIONS: {dirname}")
|
||||
broke_out = True
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
if not broke_out:
|
||||
logging.info(f"TIMED_OUT: {filef}")
|
||||
|
||||
|
||||
chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
|
||||
|
||||
try:
|
||||
os.chmod(filef, 0o777)
|
||||
except Exception as e:
|
||||
logging.error(e)
|
||||
|
||||
if decide_to_put_in_queue(filef):
|
||||
logging.info(f"QUEUE_PUT: {pfm(filef)}")
|
||||
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
|
||||
try:
|
||||
future = publish(topic, key=filef, value={"filepath": filef})
|
||||
except Exception as e:
|
||||
logging.error("KAFKA ERROR " + str(e))
|
||||
else:
|
||||
logging.info(f"QUEUE_DO_NOT_PUT: {filef}")
|
||||
|
||||
|
||||
class Handler(FileSystemEventHandler):
|
||||
def on_created(self, event):
|
||||
try:
|
||||
self.handle_it(event)
|
||||
except Exception as e:
|
||||
logging.error(e)
|
||||
|
||||
def handle_it(self, event):
|
||||
if event.is_directory:
|
||||
return
|
||||
lclass = logic_execute(event)
|
||||
lclass.start()
|
||||
|
||||
|
||||
def observe_path(path):
|
||||
event_handler = Handler()
|
||||
observer = PollingObserver()
|
||||
logger.info(f"Monitoring {path}")
|
||||
observer.schedule(event_handler, path, recursive=True)
|
||||
observer.start()
|
||||
observer.is_alive()
|
||||
return observer
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Watch and Fix Permissions And Push to Kafka Queue"
|
||||
)
|
||||
|
||||
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
paths = args.paths
|
||||
if len(paths) == 0:
|
||||
paths = default_dir_watch
|
||||
observers = [observe_path(path) for path in paths]
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
logging.error(str(e))
|
||||
|
||||
for observer in observers:
|
||||
observer.stop()
|
||||
observer.join()
|
||||
@@ -0,0 +1,150 @@
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import logging
|
||||
pfm = LogColorize.score_obj_det_embed
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip.py'
|
||||
|
||||
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
|
||||
|
||||
import sys
|
||||
sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
|
||||
from cuda_objdet_clip import infer
|
||||
from common_code import kwq, util, file_names, vector_utils
|
||||
from common_code.video_meta import FTPVideo
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import traceback
|
||||
|
||||
|
||||
client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
|
||||
|
||||
# Call logger twice to reclaim
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip'+client_suffix, stdout=True, systemd=False, level = logging.INFO)
|
||||
logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
|
||||
|
||||
|
||||
pref = ''
|
||||
priority_queue = pref + kwq.TOPICS.enter_60_videos_embed_priority
|
||||
regular_queue = pref + kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
backfill_queue = pref + kwq.TOPICS.enter_60_videos_embed_backfill
|
||||
invalid_file_topic = kwq.TOPICS.exit_60_videos_invalid
|
||||
failed_embed_topic = kwq.TOPICS.exit_60_embedding_failed
|
||||
success_embed_topic = kwq.TOPICS.exit_60_videos_embedded
|
||||
|
||||
|
||||
group_id = "vision_v3"
|
||||
client_id = "objdet_and_embedder"+client_suffix
|
||||
|
||||
|
||||
|
||||
queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
|
||||
queue_regular.subscribe([regular_queue])
|
||||
|
||||
|
||||
queue_priority = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
|
||||
queue_priority.subscribe([priority_queue])
|
||||
|
||||
queue_backfill = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
|
||||
queue_backfill.subscribe([backfill_queue])
|
||||
|
||||
|
||||
|
||||
logging.debug(f'KAFKA: Created subscribers to {regular_queue} and {priority_queue}')
|
||||
|
||||
|
||||
def check_file_validity(file_to_score):
|
||||
if not os.path.exists(file_to_score):
|
||||
return False, "file n/a"
|
||||
|
||||
if os.stat(file_to_score).st_size < 100*1000: #If less than 100kb don't score it
|
||||
return False, "file too small"
|
||||
|
||||
if '_reduced' in file_to_score:
|
||||
return False, 'file reduced resolution'
|
||||
|
||||
|
||||
if '_trimmed' in file_to_score:
|
||||
return False, 'file is trimmed'
|
||||
|
||||
if not file_to_score.endswith('.mp4'):
|
||||
return False, "Not mp4"
|
||||
|
||||
if len(FTPVideo(file_to_score).get_frames_info()) == 0:
|
||||
return False, "Failed getting frame info"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
|
||||
def get_score_file(f):
|
||||
return f.key().decode().strip('"')
|
||||
|
||||
|
||||
def perform_loop_kafka():
|
||||
score_this = None
|
||||
high_priority = False
|
||||
result_priority = queue_priority.poll(timeout=0.05)
|
||||
msg_value = None
|
||||
if result_priority is not None:
|
||||
score_this = get_score_file(result_priority)
|
||||
msg_value = result_priority.value()
|
||||
high_priority = True
|
||||
logging.info(f"KAFKA: high priority file {pfm(score_this)}")
|
||||
queue_priority.commit()
|
||||
else:
|
||||
result_regular = queue_regular.poll(timeout=0.05)
|
||||
if result_regular is not None:
|
||||
msg_value = result_regular.value()
|
||||
score_this = get_score_file(result_regular)
|
||||
logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
|
||||
queue_regular.commit()
|
||||
else:
|
||||
result_backfill = queue_backfill.poll(timeout=0.05)
|
||||
if result_backfill is not None:
|
||||
msg_value = result_backfill.value()
|
||||
score_this = get_score_file(result_backfill)
|
||||
logging.info(f"KAFKA: backfill file {pfm(score_this)}")
|
||||
queue_backfill.commit()
|
||||
|
||||
|
||||
|
||||
|
||||
if score_this is None:
|
||||
return
|
||||
|
||||
file_to_score = file_names.resolve_file_location(score_this)
|
||||
if file_to_score is None:
|
||||
return
|
||||
if file_to_score != score_this:
|
||||
logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
|
||||
|
||||
file_is_valid, message = check_file_validity(file_to_score)
|
||||
|
||||
if not file_is_valid:
|
||||
logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
|
||||
kwq.publish(invalid_file_topic, file_to_score, message)
|
||||
else:
|
||||
logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
|
||||
try:
|
||||
det_scores, clip_scores = infer.score_video_cached(file_to_score, return_dict = True)
|
||||
vector_utils.upload_vectors_to_db( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
|
||||
kwq.publish(success_embed_topic, file_to_score)
|
||||
logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
|
||||
# vector_utils.upload_to_doris( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
|
||||
logging.info(f'UPLOADED TO DB: {pfm(file_to_score)}')
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
|
||||
kwq.publish(failed_embed_topic, file_to_score, str(e))
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
perform_loop_kafka()
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import logging
|
||||
pfm = LogColorize.score_obj_det_embed
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/vision_v3/10_do_obj_det_and_clip/10_score_videos_objdet_and_clip.py'
|
||||
|
||||
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
|
||||
|
||||
import sys
|
||||
sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
|
||||
from cuda_objdet_clip import infer
|
||||
from common_code import kwq, util, file_names, vector_utils
|
||||
from common_code.video_meta import FTPVideo
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import traceback
|
||||
|
||||
|
||||
|
||||
# Call logger twice to reclaim
|
||||
logger = get_logger(__name__,'/var/log/ml_vision_logs/10_score_videos_objdet_and_clip', stdout=True, systemd=False, level = logging.INFO)
|
||||
logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
|
||||
|
||||
|
||||
pref = ''
|
||||
priority_queue = pref + kwq.TOPICS.enter_60_videos_embed_priority
|
||||
regular_queue = pref + kwq.TOPICS.exit_00_videos_to_score_detection
|
||||
backfill_queue = pref + kwq.TOPICS.enter_60_videos_embed_backfill
|
||||
invalid_file_topic = kwq.TOPICS.exit_60_videos_invalid
|
||||
failed_embed_topic = kwq.TOPICS.exit_60_embedding_failed
|
||||
success_embed_topic = kwq.TOPICS.exit_60_videos_embedded
|
||||
|
||||
client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
|
||||
|
||||
group_id = "vision_v3"
|
||||
client_id = "objdet_and_embedder"+client_suffix
|
||||
|
||||
|
||||
|
||||
queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
|
||||
queue_regular.subscribe([regular_queue])
|
||||
|
||||
|
||||
queue_priority = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
|
||||
queue_priority.subscribe([priority_queue])
|
||||
|
||||
queue_backfill = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
|
||||
queue_backfill.subscribe([backfill_queue])
|
||||
|
||||
|
||||
|
||||
logging.debug(f'KAFKA: Created subscribers to {regular_queue} and {priority_queue}')
|
||||
|
||||
|
||||
def check_file_validity(file_to_score):
|
||||
if not os.path.exists(file_to_score):
|
||||
return False, "file n/a"
|
||||
|
||||
if os.stat(file_to_score).st_size < 100*1000: #If less than 100kb don't score it
|
||||
return False, "file too small"
|
||||
|
||||
if '_reduced' in file_to_score:
|
||||
return False, 'file reduced resolution'
|
||||
|
||||
|
||||
if '_trimmed' in file_to_score:
|
||||
return False, 'file is trimmed'
|
||||
|
||||
if not file_to_score.endswith('.mp4'):
|
||||
return False, "Not mp4"
|
||||
|
||||
if len(FTPVideo(file_to_score).get_frames_info()) == 0:
|
||||
return False, "Failed getting frame info"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
|
||||
def get_score_file(f):
|
||||
return f.key().decode().strip('"')
|
||||
|
||||
|
||||
def perform_loop_kafka():
|
||||
score_this = None
|
||||
high_priority = False
|
||||
result_priority = queue_priority.poll(timeout=0.05)
|
||||
msg_value = None
|
||||
if result_priority is not None:
|
||||
score_this = get_score_file(result_priority)
|
||||
msg_value = result_priority.value()
|
||||
high_priority = True
|
||||
logging.info(f"KAFKA: high priority file {pfm(score_this)}")
|
||||
queue_priority.commit()
|
||||
else:
|
||||
result_regular = queue_regular.poll(timeout=0.05)
|
||||
if result_regular is not None:
|
||||
msg_value = result_regular.value()
|
||||
score_this = get_score_file(result_regular)
|
||||
logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
|
||||
queue_regular.commit()
|
||||
else:
|
||||
result_backfill = queue_backfill.poll(timeout=0.05)
|
||||
if result_backfill is not None:
|
||||
msg_value = result_backfill.value()
|
||||
score_this = get_score_file(result_backfill)
|
||||
logging.info(f"KAFKA: backfill file {pfm(score_this)}")
|
||||
queue_backfill.commit()
|
||||
|
||||
|
||||
|
||||
|
||||
if score_this is None:
|
||||
return
|
||||
|
||||
file_to_score = file_names.resolve_file_location(score_this)
|
||||
if file_to_score is None:
|
||||
return
|
||||
if file_to_score != score_this:
|
||||
logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
|
||||
|
||||
file_is_valid, message = check_file_validity(file_to_score)
|
||||
|
||||
if not file_is_valid:
|
||||
logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
|
||||
kwq.publish(invalid_file_topic, file_to_score, message)
|
||||
else:
|
||||
logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
|
||||
try:
|
||||
det_scores, clip_scores = infer.score_video_cached(file_to_score, return_dict = True)
|
||||
vector_utils.upload_vectors_to_db( file_to_score, clip_scores['embeds'], clip_scores['frame_numbers'])
|
||||
kwq.publish(success_embed_topic, file_to_score)
|
||||
logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
|
||||
kwq.publish(failed_embed_topic, file_to_score, str(e))
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
perform_loop_kafka()
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,4 @@
|
||||
from common_code.vector_utils import upload_vectors_to_db
|
||||
|
||||
|
||||
upload_vectors_to_db('/srv/ftp_local/backyard/2026/07/03/backyard_00_20260703192858.mp4')
|
||||
@@ -0,0 +1,57 @@
|
||||
from common_code.vector_utils import upload_vectors_to_db
|
||||
from common_code import kwq, util, file_names, vector_utils
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
import argparse
|
||||
import os
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
topic_subscribe = 'upload_to_ch_db'
|
||||
def add_files_to_queue(paths, dry_run = False, force = False, do_upload = True, do_queue = False):
|
||||
queued = set()
|
||||
for rt in paths:
|
||||
for root, dirs, files in os.walk(rt):
|
||||
for f in files:
|
||||
new_path = os.path.join(root, f)
|
||||
if new_path.endswith('detection.npz') and 'leopards' not in new_path.lower():
|
||||
# if decide_to_put_in_queue(new_path, force = force):
|
||||
queued.add(new_path)
|
||||
|
||||
for x in tqdm(queued):
|
||||
try:
|
||||
if do_upload:
|
||||
upload_vectors_to_db(x)
|
||||
if do_queue:
|
||||
kwq.publish(topic_subscribe, x, x)
|
||||
|
||||
except Exception as e:
|
||||
print('Failed: '+str(x)+'with '+str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Watch and Fix Permissions And Push to Kafka Queue"
|
||||
)
|
||||
|
||||
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
|
||||
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
|
||||
parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
|
||||
parser.add_argument("--skip_upload", action='store_true', help='Force adding (do not check if already done)')
|
||||
parser.add_argument("--do_queue", action='store_true', help='Force adding (do not check if already done)')
|
||||
args, _ = parser.parse_known_args()
|
||||
paths = args.paths
|
||||
force = args.force
|
||||
dry_run = args.dry_run
|
||||
do_queue = args.do_queue
|
||||
do_upload = not args.skip_upload
|
||||
if len(paths) == 0:
|
||||
paths = default_dir_watch
|
||||
|
||||
add_files_to_queue(paths, dry_run = dry_run, force=force, do_upload = do_upload, do_queue = do_queue)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from common_code.vector_utils import upload_to_doris
|
||||
from tqdm import tqdm
|
||||
import argparse
|
||||
import os
|
||||
|
||||
def add_files_to_queue(paths, dry_run = False, force = False):
|
||||
queued = set()
|
||||
for rt in paths:
|
||||
for root, dirs, files in os.walk(rt):
|
||||
for f in files:
|
||||
new_path = os.path.join(root, f)
|
||||
if new_path.endswith('.oclip_embeds.npz'):
|
||||
# if decide_to_put_in_queue(new_path, force = force):
|
||||
queued.add(new_path)
|
||||
|
||||
for x in tqdm(queued):
|
||||
try:
|
||||
upload_to_doris(x)
|
||||
except Exception as e:
|
||||
print('Failed: '+str(x)+'with '+str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Watch and Fix Permissions And Push to Kafka Queue"
|
||||
)
|
||||
|
||||
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
|
||||
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
|
||||
parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
|
||||
args, _ = parser.parse_known_args()
|
||||
paths = args.paths
|
||||
force = args.force
|
||||
dry_run = args.dry_run
|
||||
|
||||
if len(paths) == 0:
|
||||
paths = default_dir_watch
|
||||
|
||||
add_files_to_queue(paths, dry_run = dry_run, force=force)
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from common_code.vector_utils import upload_vectors_to_db
|
||||
from common_code import kwq, util, file_names, vector_utils, settings
|
||||
from multiprocessing import Pool
|
||||
from tqdm import tqdm
|
||||
import argparse
|
||||
import os
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
topic_subscribe = 'upload_to_vector_db'
|
||||
def add_files_to_queue(paths, dry_run = False, force = False, do_upload = True, do_queue = False):
|
||||
queued = set()
|
||||
for rt in paths:
|
||||
for root, dirs, files in os.walk(rt):
|
||||
for f in files:
|
||||
new_path = os.path.join(root, f)
|
||||
if new_path.endswith('.oclip_embeds.npz'):
|
||||
# if decide_to_put_in_queue(new_path, force = force):
|
||||
queued.add(new_path)
|
||||
|
||||
for x in tqdm(queued):
|
||||
try:
|
||||
if do_upload:
|
||||
if os.path.exists(x+'.in_queue'):
|
||||
pass
|
||||
else:
|
||||
upload_vectors_to_db(x)
|
||||
with open(x+'.in_queue','w') as ff:
|
||||
pass
|
||||
if do_queue:
|
||||
kwq.publish(topic_subscribe, x, x)
|
||||
|
||||
except Exception as e:
|
||||
print('Failed: '+str(x)+'with '+str(e))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Watch and Fix Permissions And Push to Kafka Queue"
|
||||
)
|
||||
|
||||
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
|
||||
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
|
||||
parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
|
||||
parser.add_argument("--skip_upload", action='store_true', help='Force adding (do not check if already done)')
|
||||
parser.add_argument("--do_queue", action='store_true', help='Force adding (do not check if already done)')
|
||||
args, _ = parser.parse_known_args()
|
||||
paths = args.paths
|
||||
force = args.force
|
||||
dry_run = args.dry_run
|
||||
do_queue = args.do_queue
|
||||
do_upload = not args.skip_upload
|
||||
if len(paths) == 0:
|
||||
paths = settings.dir_watch
|
||||
|
||||
add_files_to_queue(paths, dry_run = dry_run, force=force, do_upload = do_upload, do_queue = do_queue)
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
|
||||
import numpy as np
|
||||
import time
|
||||
from multiprocessing import Process, Queue, Manager
|
||||
import threading
|
||||
from common_code import kwq, util, file_names, vector_utils, video_meta
|
||||
|
||||
from datetime import datetime
|
||||
from clickhouse_connect import get_client
|
||||
|
||||
|
||||
topic_subscribe = 'upload_to_ch_db'
|
||||
group_id = "vision_v3_uploaders"
|
||||
|
||||
def get_score_file(f):
|
||||
return f.key().decode().strip('"')
|
||||
|
||||
|
||||
|
||||
def insert_into_ch(file_path, client):
|
||||
cset = util.get_cset_match(file_path)
|
||||
ff = np.load(cset['.detection.npz'], allow_pickle = True)
|
||||
det_results = dict(ff)
|
||||
del_idx = list()
|
||||
|
||||
filt_scores = det_results['final_scores']
|
||||
filt_frames = det_results['final_frames']
|
||||
spec_label_idx = ff['species_label_map'].tolist()
|
||||
|
||||
ftp_vid = video_meta.FTPVideo(cset)
|
||||
|
||||
frame_num_off = ftp_vid.sorted_frames_info
|
||||
frame_period = ftp_vid.frameperiod
|
||||
frame_rate = ftp_vid.framerate
|
||||
|
||||
indexes = np.searchsorted(ff['scored_frames'], filt_frames)
|
||||
dff = np.diff(ff['scored_frames'])
|
||||
|
||||
head = np.insert(dff, 0, 0)
|
||||
tail = np.append(dff, 0)
|
||||
all_frame_durations = frame_period*np.mean(np.stack([head,tail]), axis=0)
|
||||
|
||||
|
||||
frame_file = [ftp_vid.cpath.split('/')[-1].replace('_reduced','').replace('.mp4','')]*len(filt_frames)
|
||||
frame_bboxes = ff['final_boxes']
|
||||
frame_numbers = filt_frames
|
||||
frame_scores = filt_scores
|
||||
frame_species = [spec_label_idx[int(x)] for x in ff['final_labels']]
|
||||
frame_durations = all_frame_durations[indexes]
|
||||
frame_tstamps = [frame_num_off[x]['time'] for x in filt_frames]
|
||||
db_insert_times = [datetime.now()]*len(filt_frames)
|
||||
|
||||
a = frame_numbers
|
||||
starts = np.r_[0, np.flatnonzero(a[1:] != a[:-1]) + 1]
|
||||
det_count_num = np.arange(len(a)) - np.repeat(starts, np.diff(np.r_[starts, len(a)]))
|
||||
camera = [ftp_vid.camera_name] * len(filt_frames)
|
||||
|
||||
|
||||
result = client.insert('nuggets.cameras_object_detection_results',
|
||||
column_oriented = True,
|
||||
column_names = ['camera','video_filename','frame_number','detection_number','classification_score','object_name','bbox_xyxy','frame_duration','frame_timestamp','db_insert_time'],
|
||||
data = [camera, frame_file, frame_numbers, det_count_num, frame_scores, frame_species, frame_bboxes.tolist(), frame_durations, frame_tstamps, db_insert_times]
|
||||
)
|
||||
|
||||
|
||||
# %%
|
||||
|
||||
|
||||
def watch_and_upload(rank, health_queue):
|
||||
client = get_client(host='192.168.1.242', compression=True)
|
||||
start_time = time.time()
|
||||
client_suffix = '_'+str(rank)
|
||||
client_id = 'uploader_to_ch_db'+client_suffix
|
||||
current_time = time.time()
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
|
||||
|
||||
queue = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
|
||||
queue.subscribe([topic_subscribe])
|
||||
|
||||
last_heartbeat = time.time()
|
||||
|
||||
while True:
|
||||
current_time = time.time()
|
||||
|
||||
# # Exit after 5 minutes
|
||||
# if (current_time - start_time) > 60*5:
|
||||
# print('###########################')
|
||||
# print('######### EXITING #########')
|
||||
# print('###########################')
|
||||
# return
|
||||
|
||||
# Send heartbeat every 10 seconds
|
||||
if current_time - last_heartbeat > 5:
|
||||
try:
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
|
||||
last_heartbeat = current_time
|
||||
except:
|
||||
# Queue might be full, continue working
|
||||
pass
|
||||
|
||||
result = queue.poll(timeout=0.05)
|
||||
if result is not None:
|
||||
try:
|
||||
score_this = get_score_file(result)
|
||||
insert_into_ch(score_this, client)
|
||||
print('#' + str(rank) + ' Uploaded to ' + score_this)
|
||||
# Send success heartbeat
|
||||
try:
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'working'})
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
print('Failed ' + score_this + ':' + str(e))
|
||||
|
||||
queue.commit()
|
||||
|
||||
def health_monitor(processes, health_queues, num_workers):
|
||||
"""Monitor process health and restart hung processes"""
|
||||
last_heartbeat = {}
|
||||
HEALTH_TIMEOUT = 120 # Consider process hung if no heartbeat for 60 seconds
|
||||
|
||||
while True:
|
||||
time.sleep(5) # Check every 5 seconds
|
||||
current_time = time.time()
|
||||
|
||||
for rank in range(num_workers):
|
||||
# Drain the health queue for this process
|
||||
latest_heartbeat = None
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
heartbeat = health_queues[rank].get_nowait()
|
||||
if heartbeat['rank'] == rank:
|
||||
latest_heartbeat = heartbeat['timestamp']
|
||||
except:
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
# Update last known heartbeat
|
||||
if latest_heartbeat:
|
||||
last_heartbeat[rank] = latest_heartbeat
|
||||
|
||||
# Check if process is alive and responsive
|
||||
process = processes[rank]
|
||||
last_seen = last_heartbeat.get(rank, 0)
|
||||
|
||||
if not process.is_alive():
|
||||
print(f"Process {rank} has died. Restarting...")
|
||||
processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
processes[rank].start()
|
||||
last_heartbeat[rank] = current_time
|
||||
elif current_time - last_seen > HEALTH_TIMEOUT:
|
||||
print(f"Process {rank} appears hung (no heartbeat for {current_time - last_seen:.1f}s). Restarting...")
|
||||
process.terminate()
|
||||
process.join(timeout=5)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
|
||||
# Create new process
|
||||
processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
processes[rank].start()
|
||||
last_heartbeat[rank] = current_time
|
||||
|
||||
# Main execution
|
||||
if __name__ == "__main__":
|
||||
num_workers = 4
|
||||
|
||||
# Create health queues for each worker
|
||||
health_queues = [Queue(maxsize=10) for _ in range(num_workers)]
|
||||
|
||||
# Create and start worker processes
|
||||
processes = []
|
||||
for rank in range(num_workers):
|
||||
p = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
p.start()
|
||||
processes.append(p)
|
||||
|
||||
# Start health monitor in a separate thread
|
||||
monitor_thread = threading.Thread(
|
||||
target=health_monitor,
|
||||
args=(processes, health_queues, num_workers),
|
||||
daemon=True
|
||||
)
|
||||
monitor_thread.start()
|
||||
|
||||
try:
|
||||
# Wait for all processes to complete (they will exit after 5 minutes)
|
||||
for p in processes:
|
||||
p.join()
|
||||
print("All worker processes completed")
|
||||
except KeyboardInterrupt:
|
||||
print("Shutting down...")
|
||||
for p in processes:
|
||||
p.terminate()
|
||||
p.join(timeout=5)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
p.join()
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
from common_code.vector_utils import upload_vectors_to_db
|
||||
from common_code import kwq, util, file_names, vector_utils
|
||||
import time
|
||||
from multiprocessing import Process, Queue, Manager
|
||||
import threading
|
||||
|
||||
topic_subscribe = 'upload_to_vector_db'
|
||||
group_id = "vision_v3_uploaders"
|
||||
|
||||
def get_score_file(f):
|
||||
return f.key().decode().strip('"')
|
||||
|
||||
def watch_and_upload(rank, health_queue):
|
||||
start_time = time.time()
|
||||
client_suffix = '_'+str(rank)
|
||||
client_id = 'uploader_to_vector_db'+client_suffix
|
||||
|
||||
queue = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'earliest')
|
||||
queue.subscribe([topic_subscribe])
|
||||
|
||||
last_heartbeat = time.time()
|
||||
|
||||
while True:
|
||||
current_time = time.time()
|
||||
|
||||
# # Exit after 5 minutes
|
||||
# if (current_time - start_time) > 60*5:
|
||||
# print('###########################')
|
||||
# print('######### EXITING #########')
|
||||
# print('###########################')
|
||||
# return
|
||||
|
||||
# Send heartbeat every 10 seconds
|
||||
if current_time - last_heartbeat > 5:
|
||||
try:
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'alive'})
|
||||
last_heartbeat = current_time
|
||||
except:
|
||||
# Queue might be full, continue working
|
||||
pass
|
||||
|
||||
result = queue.poll(timeout=0.05)
|
||||
if result is not None:
|
||||
score_this = get_score_file(result)
|
||||
try:
|
||||
upload_vectors_to_db(score_this)
|
||||
print('#' + str(rank) + ' Uploaded to ' + score_this)
|
||||
# Send success heartbeat
|
||||
try:
|
||||
health_queue.put({'rank': rank, 'timestamp': current_time, 'status': 'working'})
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
print('Failed ' + score_this + ':' + str(e))
|
||||
|
||||
queue.commit()
|
||||
|
||||
def health_monitor(processes, health_queues, num_workers):
|
||||
"""Monitor process health and restart hung processes"""
|
||||
last_heartbeat = {}
|
||||
HEALTH_TIMEOUT = 15 # Consider process hung if no heartbeat for 60 seconds
|
||||
|
||||
while True:
|
||||
time.sleep(5) # Check every 5 seconds
|
||||
current_time = time.time()
|
||||
|
||||
for rank in range(num_workers):
|
||||
# Drain the health queue for this process
|
||||
latest_heartbeat = None
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
heartbeat = health_queues[rank].get_nowait()
|
||||
if heartbeat['rank'] == rank:
|
||||
latest_heartbeat = heartbeat['timestamp']
|
||||
except:
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
# Update last known heartbeat
|
||||
if latest_heartbeat:
|
||||
last_heartbeat[rank] = latest_heartbeat
|
||||
|
||||
# Check if process is alive and responsive
|
||||
process = processes[rank]
|
||||
last_seen = last_heartbeat.get(rank, 0)
|
||||
|
||||
if not process.is_alive():
|
||||
print(f"Process {rank} has died. Restarting...")
|
||||
processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
processes[rank].start()
|
||||
last_heartbeat[rank] = current_time
|
||||
elif current_time - last_seen > HEALTH_TIMEOUT:
|
||||
print(f"Process {rank} appears hung (no heartbeat for {current_time - last_seen:.1f}s). Restarting...")
|
||||
process.terminate()
|
||||
process.join(timeout=5)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
|
||||
# Create new process
|
||||
processes[rank] = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
processes[rank].start()
|
||||
last_heartbeat[rank] = current_time
|
||||
|
||||
# Main execution
|
||||
if __name__ == "__main__":
|
||||
num_workers = 4
|
||||
|
||||
# Create health queues for each worker
|
||||
health_queues = [Queue(maxsize=10) for _ in range(num_workers)]
|
||||
|
||||
# Create and start worker processes
|
||||
processes = []
|
||||
for rank in range(num_workers):
|
||||
p = Process(target=watch_and_upload, args=(rank, health_queues[rank]))
|
||||
p.start()
|
||||
processes.append(p)
|
||||
|
||||
# Start health monitor in a separate thread
|
||||
monitor_thread = threading.Thread(
|
||||
target=health_monitor,
|
||||
args=(processes, health_queues, num_workers),
|
||||
daemon=True
|
||||
)
|
||||
monitor_thread.start()
|
||||
|
||||
try:
|
||||
# Wait for all processes to complete (they will exit after 5 minutes)
|
||||
for p in processes:
|
||||
p.join()
|
||||
print("All worker processes completed")
|
||||
except KeyboardInterrupt:
|
||||
print("Shutting down...")
|
||||
for p in processes:
|
||||
p.terminate()
|
||||
p.join(timeout=5)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
p.join()
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
from common_code.settings import get_logger, LogColorize
|
||||
import logging
|
||||
pfm = LogColorize.score_obj_det_embed
|
||||
if not ('__file__' in vars() or '__file__' in globals()):
|
||||
__file__ = '/home/thebears/Source/pipelines/vision_v3/40_do_cropped_clip/40_do_cropped_clip.py'
|
||||
|
||||
|
||||
log_file = '/var/log/ml_vision_logs/40_do_cropped_clip'
|
||||
logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
|
||||
|
||||
import sys
|
||||
sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
|
||||
from cuda_objdet_clip import infer
|
||||
from common_code import kwq, util, file_names, vector_utils
|
||||
from common_code.video_meta import FTPVideo
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import traceback
|
||||
|
||||
|
||||
client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
|
||||
|
||||
# Call logger twice to reclaim
|
||||
logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
|
||||
logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
|
||||
|
||||
|
||||
pref = ''
|
||||
regular_queue = pref + kwq.TOPICS.exit_60_videos_embedded
|
||||
|
||||
|
||||
|
||||
group_id = "vision_v3"
|
||||
client_id = "cropped_embedder"+client_suffix
|
||||
|
||||
|
||||
queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
|
||||
queue_regular.subscribe([regular_queue])
|
||||
|
||||
logging.debug(f'KAFKA: Created subscribers to {regular_queue}')
|
||||
|
||||
def check_file_validity(file_to_score):
|
||||
if 'leopard' in file_to_score.lower():
|
||||
return True, ''
|
||||
else:
|
||||
return False, ''
|
||||
|
||||
|
||||
def get_score_file(f):
|
||||
return f.key().decode().strip('"')
|
||||
|
||||
def perform_loop_kafka():
|
||||
score_this = None
|
||||
msg_value = None
|
||||
result_regular = queue_regular.poll(timeout=0.05)
|
||||
if result_regular is not None:
|
||||
msg_value = result_regular.value()
|
||||
score_this = get_score_file(result_regular)
|
||||
logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
|
||||
queue_regular.commit()
|
||||
|
||||
if score_this is None:
|
||||
return
|
||||
|
||||
file_to_score = file_names.resolve_file_location(score_this)
|
||||
if file_to_score is None:
|
||||
return
|
||||
|
||||
|
||||
|
||||
|
||||
if file_to_score != score_this:
|
||||
logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
|
||||
|
||||
file_is_valid, message = check_file_validity(file_to_score)
|
||||
if not file_is_valid:
|
||||
logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
|
||||
return
|
||||
|
||||
logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
|
||||
try:
|
||||
infer.score_video_sub_clip_cached( file_to_score)
|
||||
logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
|
||||
|
||||
while True:
|
||||
perform_loop_kafka()
|
||||
|
||||
Reference in New Issue
Block a user