YACWC

vision_v3/40_do_cropped_clip/40_do_cropped_clip.py

@@ -0,0 +1,92 @@
+from common_code.settings import get_logger, LogColorize
+import logging
+pfm = LogColorize.score_obj_det_embed
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/vision_v3/40_do_cropped_clip/40_do_cropped_clip.py'
+
+
+log_file = '/var/log/ml_vision_logs/40_do_cropped_clip'
+logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
+
+import sys
+sys.path.insert(0,'/home/thebears/Source/task_runners/vision_v3')
+from cuda_objdet_clip import infer
+from common_code import kwq, util, file_names, vector_utils
+from common_code.video_meta import FTPVideo
+import os
+import sys
+import time
+import json
+import traceback
+
+
+client_suffix = os.environ.get('CONSUMER_CLIENT_SUFFIX','')
+
+# Call logger twice to reclaim
+logger = get_logger(__name__,log_file, stdout=True, systemd=False, level = logging.INFO)
+logger.info(pfm(f"Starting objdet_and_embedder scoring @ {__file__}"))
+
+
+pref = ''
+regular_queue = pref + kwq.TOPICS.exit_60_videos_embedded
+
+
+
+group_id = "vision_v3"
+client_id = "cropped_embedder"+client_suffix
+
+
+queue_regular = kwq.create_consumer(group_id = group_id, client_id = client_id, auto_offset_reset = 'latest')
+queue_regular.subscribe([regular_queue])
+
+logging.debug(f'KAFKA: Created subscribers to {regular_queue}')
+
+def check_file_validity(file_to_score):
+    if 'leopard' in file_to_score.lower():
+        return True, ''
+    else:
+        return False, ''
+    
+
+def get_score_file(f):
+    return f.key().decode().strip('"')
+
+def perform_loop_kafka():
+    score_this = None
+    msg_value = None
+    result_regular = queue_regular.poll(timeout=0.05)
+    if result_regular is not None:
+        msg_value = result_regular.value()            
+        score_this = get_score_file(result_regular)
+        logging.info(f"KAFKA: regular priority file {pfm(score_this)}")
+        queue_regular.commit()
+    
+    if score_this is None:
+        return
+
+    file_to_score = file_names.resolve_file_location(score_this)
+    if file_to_score is None:
+        return
+
+    
+    
+    
+    if file_to_score != score_this:
+        logger.info(f"FILE_PATH_CHANGED: from {pfm(score_this)} to {pfm(file_to_score)}")
+
+    file_is_valid, message = check_file_validity(file_to_score)
+    if not file_is_valid:
+        logger.info(f'INVALID_FILE: Skipping {pfm(file_to_score)}')
+        return
+
+    logger.info(f'VALID_FILE: Scoring {pfm(file_to_score)}')
+    try:
+        infer.score_video_sub_clip_cached( file_to_score)
+        logger.info(f'SCORING_SUCCESS: {pfm(file_to_score)}')
+    except Exception as e:
+        traceback.print_exc()
+        logging.error(f"SCORING_ERROR: {pfm(file_to_score)}: {e}")
+
+while True:
+    perform_loop_kafka()
+