YACWC

00_watch_for_new_videos/load_missed.py

@@ -1,4 +1,5 @@
 import os
+
 import subprocess as sp
 import time
 import psutil
@@ -12,6 +13,14 @@ from CommonCode import wq
 from CommonCode import kwq
 from CommonCode.settings import get_logger, LogColorize
 import argparse
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
+
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(parent_dir)
+logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False)
+from common import settings
+default_dir_watch = settings.dir_watch
 
 
 publish = kwq.producer.send
@@ -28,7 +37,8 @@ def decide_to_put_in_queue(file_name):
         return True
     else:
         return False
-
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(parent_dir)
 def place_file_in_queue(filef):
     dirname = os.path.dirname(filef)
     foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
@@ -39,18 +49,20 @@ def place_file_in_queue(filef):
 
     chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
     try:
+        
         os.chmod(filef, 0o777)
     except Exception as e:
-        logging.error(e)
+        pass
+#$        logging.error(e)
 
     logging.info(f"QUEUE_PUT: {filef}")
     publish(topic, key=filef, value={"filepath": filef})
+    wq.publish(wq.TOPICS.ml_vision_to_score, filef)
 
 
-# %%
-paths = ['/srv/ftp/sidefeeder','/srv/ftp/ptz']
-queued = set()
-for rt in paths:
+def add_files_to_queue(paths):
+    queued = set()
+    for rt in paths:
         for root, dirs, files in os.walk(rt):
             for f in files:
                 new_path = os.path.join(root, f)
@@ -58,7 +70,7 @@ for rt in paths:
                     queued.add(new_path)
 
 
-for x in queued:
+    for x in queued:
         place_file_in_queue(x)
 
     
@@ -67,9 +79,15 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         prog="Watch and Fix Permissions And Push to Kafka Queue"
     )
-    parser.add_argument("paths", nargs="+", help="Paths to monitor")
-    args = parser.parse_args()
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+
+    args, _ = parser.parse_known_args()
     paths = args.paths
 
+    if len(paths) == 0:
+        paths = default_dir_watch
+
+    add_files_to_queue(paths)
 
     

00_watch_for_new_videos/watch_and_fix_permissions.py

@@ -13,7 +13,15 @@ from CommonCode import kwq
 from CommonCode.settings import get_logger, LogColorize
 import argparse
 pfm = LogColorize.watch_and_fix_permissions
+if not ('__file__' in vars() or '__file__' in globals()):
+    __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
 
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(parent_dir)
+
+from common import settings
+default_dir_watch = settings.dir_watch
+# %%
 logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False)
 
 logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
@@ -126,9 +134,13 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         prog="Watch and Fix Permissions And Push to Kafka Queue"
     )
-    parser.add_argument("paths", nargs="+", help="Paths to monitor")
-    args = parser.parse_args()
+
+    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
+
+    args, _ = parser.parse_known_args()
     paths = args.paths
+    if len(paths) == 0:
+        paths = default_dir_watch
     observers = [observe_path(path) for path in paths]
     
     try:

02_remove_without_nuggets/remove_without_nuggets.py

@@ -33,9 +33,31 @@ class RESULT_TYPE():
     NO_JSON = 4
 
 
+def get_ok_to_delete(file_path):
+    max_look = 5
+    c_path = file_path
+    settings_default = {'class_threshold':10000, 'frames_with_dets_threshold': -10000}
+    for i in range(max_look):
+        c_path  = os.path.abspath(os.path.join(c_path, '..'))
+        test_path = os.path.join(c_path, 'settings.json')
+        if os.path.exists(test_path):
+            with open(test_path, 'r') as rr:
+                settings = json.load(rr)
+                settings_default.update(settings)
+
+    return settings_default
+
 def exec_file_remove_logic(file_path):
     result = RESULT_TYPE.NOTHING
     cset = get_cset_for_file_matching(file_path)
+
+    settings = get_ok_to_delete(cset['.mp4'])
+    logger.info(f"EVALUATING LOGIC WITH SETTINGS: "+str(settings))
+    if settings['class_threshold'] > 1:
+        logger.info(f"THRESHOLD SET TO ABOVE 1, SKIPPING: {file_path}")
+        result = RESULT_TYPE.HAS_OBJS
+        return
+    
     logger.info(f"EXECUTING_LOGIC :{file_path}")
     json_check = '.json.orin'
     if '.has_objs' in cset:
@@ -52,8 +74,8 @@ def exec_file_remove_logic(file_path):
             with open(cset[json_check],'r') as jj:
                 det_data = json.load(jj)
 
-        thresh = 0.2
-        frames_with_dets_thresh = 0.1
+        thresh = settings['class_threshold']
+        frames_with_dets_thresh = settings['frames_with_dets_threshold']
 
         all_scores = list()
         for x in det_data['scores']:

common/settings.py

@@ -4,6 +4,6 @@ dir_watch = [
     "/srv/ftp/railing/",
     "/srv/ftp/sidefeeder",
     "/srv/ftp_tcc/leopards1",
-    "/srv/ftp_tcc/leopard",
+    "/srv/ftp_tcc/leopards2",
 ]