From 571500b291fcde8f9a55d164c998fac288cdff7b Mon Sep 17 00:00:00 2001 From: "Ishan S. Patel" Date: Fri, 18 Jul 2025 17:15:50 -0400 Subject: [PATCH] YACWC --- 00_watch_for_new_videos/load_missed.py | 48 ++++++++++++------ .../watch_and_fix_permissions.py | 16 +++++- .../remove_without_nuggets.py | 26 +++++++++- common/__pycache__/settings.cpython-312.pyc | Bin 0 -> 296 bytes common/__pycache__/settings.cpython-313.pyc | Bin 0 -> 296 bytes common/settings.py | 2 +- 6 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 common/__pycache__/settings.cpython-312.pyc create mode 100644 common/__pycache__/settings.cpython-313.pyc diff --git a/00_watch_for_new_videos/load_missed.py b/00_watch_for_new_videos/load_missed.py index 730fae8..5002594 100755 --- a/00_watch_for_new_videos/load_missed.py +++ b/00_watch_for_new_videos/load_missed.py @@ -1,4 +1,5 @@ import os + import subprocess as sp import time import psutil @@ -12,6 +13,14 @@ from CommonCode import wq from CommonCode import kwq from CommonCode.settings import get_logger, LogColorize import argparse +if not ('__file__' in vars() or '__file__' in globals()): + __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py' + +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(parent_dir) +logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False) +from common import settings +default_dir_watch = settings.dir_watch publish = kwq.producer.send @@ -28,7 +37,8 @@ def decide_to_put_in_queue(file_name): return True else: return False - +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(parent_dir) def place_file_in_queue(filef): dirname = os.path.dirname(filef) foldmode = stat.S_IMODE(os.stat(dirname).st_mode) @@ -39,27 +49,29 @@ def place_file_in_queue(filef): chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)} try: + os.chmod(filef, 0o777) except Exception as e: - logging.error(e) + pass +#$ logging.error(e) logging.info(f"QUEUE_PUT: {filef}") publish(topic, key=filef, value={"filepath": filef}) + wq.publish(wq.TOPICS.ml_vision_to_score, filef) -# %% -paths = ['/srv/ftp/sidefeeder','/srv/ftp/ptz'] -queued = set() -for rt in paths: - for root, dirs, files in os.walk(rt): - for f in files: - new_path = os.path.join(root, f) - if decide_to_put_in_queue(new_path): - queued.add(new_path) +def add_files_to_queue(paths): + queued = set() + for rt in paths: + for root, dirs, files in os.walk(rt): + for f in files: + new_path = os.path.join(root, f) + if decide_to_put_in_queue(new_path): + queued.add(new_path) -for x in queued: - place_file_in_queue(x) + for x in queued: + place_file_in_queue(x) # %% @@ -67,9 +79,15 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( prog="Watch and Fix Permissions And Push to Kafka Queue" ) - parser.add_argument("paths", nargs="+", help="Paths to monitor") - args = parser.parse_args() + + parser.add_argument("paths", nargs="*", help="Paths to monitor", default=()) + + args, _ = parser.parse_known_args() paths = args.paths + if len(paths) == 0: + paths = default_dir_watch + + add_files_to_queue(paths) diff --git a/00_watch_for_new_videos/watch_and_fix_permissions.py b/00_watch_for_new_videos/watch_and_fix_permissions.py index 13e54d2..82a1c8d 100755 --- a/00_watch_for_new_videos/watch_and_fix_permissions.py +++ b/00_watch_for_new_videos/watch_and_fix_permissions.py @@ -13,7 +13,15 @@ from CommonCode import kwq from CommonCode.settings import get_logger, LogColorize import argparse pfm = LogColorize.watch_and_fix_permissions +if not ('__file__' in vars() or '__file__' in globals()): + __file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py' +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(parent_dir) + +from common import settings +default_dir_watch = settings.dir_watch +# %% logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False) logger.info(pfm(f"Starting watch_and_fix_permissions.py")) @@ -126,9 +134,13 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( prog="Watch and Fix Permissions And Push to Kafka Queue" ) - parser.add_argument("paths", nargs="+", help="Paths to monitor") - args = parser.parse_args() + + parser.add_argument("paths", nargs="*", help="Paths to monitor", default=()) + + args, _ = parser.parse_known_args() paths = args.paths + if len(paths) == 0: + paths = default_dir_watch observers = [observe_path(path) for path in paths] try: diff --git a/02_remove_without_nuggets/remove_without_nuggets.py b/02_remove_without_nuggets/remove_without_nuggets.py index 4d0d869..e7dfbd4 100755 --- a/02_remove_without_nuggets/remove_without_nuggets.py +++ b/02_remove_without_nuggets/remove_without_nuggets.py @@ -33,9 +33,31 @@ class RESULT_TYPE(): NO_JSON = 4 +def get_ok_to_delete(file_path): + max_look = 5 + c_path = file_path + settings_default = {'class_threshold':10000, 'frames_with_dets_threshold': -10000} + for i in range(max_look): + c_path = os.path.abspath(os.path.join(c_path, '..')) + test_path = os.path.join(c_path, 'settings.json') + if os.path.exists(test_path): + with open(test_path, 'r') as rr: + settings = json.load(rr) + settings_default.update(settings) + + return settings_default + def exec_file_remove_logic(file_path): result = RESULT_TYPE.NOTHING cset = get_cset_for_file_matching(file_path) + + settings = get_ok_to_delete(cset['.mp4']) + logger.info(f"EVALUATING LOGIC WITH SETTINGS: "+str(settings)) + if settings['class_threshold'] > 1: + logger.info(f"THRESHOLD SET TO ABOVE 1, SKIPPING: {file_path}") + result = RESULT_TYPE.HAS_OBJS + return + logger.info(f"EXECUTING_LOGIC :{file_path}") json_check = '.json.orin' if '.has_objs' in cset: @@ -52,8 +74,8 @@ def exec_file_remove_logic(file_path): with open(cset[json_check],'r') as jj: det_data = json.load(jj) - thresh = 0.2 - frames_with_dets_thresh = 0.1 + thresh = settings['class_threshold'] + frames_with_dets_thresh = settings['frames_with_dets_threshold'] all_scores = list() for x in det_data['scores']: diff --git a/common/__pycache__/settings.cpython-312.pyc b/common/__pycache__/settings.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cdb4d8aca3498b3c6262ee234454ea6071c7cd20 GIT binary patch literal 296 zcmX@j%ge>Uz`*ddxhi8L0|Ucj5C?{tpp4Hv3=9m@8PXY27#1-`F;p^YGHJ3^iRc#> zmFcII6zFG^=H_PRr6*+;rBw02MGH!*ss!P@qQuOc%)E5{Dq*-#ab`+tT54)aYEhLK zOfbGAIaxm^HNPOSD5cmChq#fSCgUy6l+2>|^2CzljFk+ZK@R<8q@R(Wo2p-uk(!j6 zSX8VZoL^d$oT^`tS&*8OnU`9upPZkYo1dp&oLW+nnU`LyS5SG2!zMRBr8Fniu856+ efq{{MfuR`W*$>Q&jEtX{*m;>6Ig3~r7#IKoJ6Y-g literal 0 HcmV?d00001 diff --git a/common/__pycache__/settings.cpython-313.pyc b/common/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8cc515b6adcf0c58fae4cfc94c08de0006d67407 GIT binary patch literal 296 zcmey&%ge>Uz`*ddxhi8L0|Ucj5C?{tpp4Hv3=9lY8T1)~83GxL7}6OvnKap|MD&Y` z%JkDp3iLBdb8|EE(vvcaQmT01q6H;YRf2F{QDSCJW?s5}l`veWI5QPAw_P%u6rUE2zB1VUwGmQks)$SH#A^ ez`)4Bz)%eG><4B>M#fJ}?7U2koJA}Q3=9C0hFM$y literal 0 HcmV?d00001 diff --git a/common/settings.py b/common/settings.py index 534c496..6f862bf 100644 --- a/common/settings.py +++ b/common/settings.py @@ -4,6 +4,6 @@ dir_watch = [ "/srv/ftp/railing/", "/srv/ftp/sidefeeder", "/srv/ftp_tcc/leopards1", - "/srv/ftp_tcc/leopard", + "/srv/ftp_tcc/leopards2", ]