thebears/pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YACWC

Browse Source
This commit is contained in:
Ishan S. Patel
2026-07-08 10:53:37 -04:00
parent 19ace202ba
commit d286b9b311
25 changed files with 527322 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vision_v3/00_watch_for_new_videos/00_load_missed.py
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/home/thebears/envs/ml_vision_objdet/bin/python
import os
import subprocess as sp
import time
import psutil
import stat
import sys
import logging
from watchdog.observers.polling import PollingObserver
from watchdog.events import FileSystemEventHandler
from threading import Thread
from tqdm import tqdm
from common_code import kwq
from common_code.settings import get_logger, LogColorize
import argparse
if not ('__file__' in vars() or '__file__' in globals()):
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir)
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
from common_code import settings
#default_dir_watch = settings.dir_watch_missed
default_dir_watch = settings.dir_watch
publish = kwq.producer.send
#topic = kwq.TOPICS.exit_00_videos_to_score_detection
topic = kwq.TOPICS.enter_60_videos_embed_backfill
necessary_extensions = [
'.oclip_embeds.npz',
'.detection.npz']
def decide_to_put_in_queue(file_name, force = False):
if os.path.getsize(file_name) < 1*1000*1000:
return False
disqualifiers = list()
disqualifiers.append(not os.path.exists(file_name))
disqualifiers.append("_reduced" in file_name)
disqualifiers.append("_trimmed" in file_name)
# disqualifiers.append('hummingbird' in file_name)
disqualified = any(disqualifiers)
if file_name.endswith(".mp4") and not disqualified:
rt_path, _ = os.path.splitext(file_name)
if force:
all_exist = False
else:
all_exist = all([os.path.exists( rt_path + ext) for ext in necessary_extensions])
return (not all_exist)
else:
return False
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir)
def place_file_in_queue(filef):
dirname = os.path.dirname(filef)
foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
if oct(foldmode) != "0o777":
os.system(f"sudo chmod 777 {dirname}")
logging.info(f"SETTING_PERMISSIONS: {dirname}")
chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
try:
os.chmod(filef, 0o777)
except Exception as e:
pass
#$ logging.error(e)
logging.info(f"QUEUE_PUT: {filef}")
publish(topic, key=filef, value={"filepath": filef})
def add_files_to_queue(paths, dry_run = False, force = False):
queued = set()
for rt in paths:
for root, dirs, files in os.walk(rt):
for f in files:
new_path = os.path.join(root, f)
if decide_to_put_in_queue(new_path, force = force):
queued.add(new_path)
for x in tqdm(queued):
if dry_run:
print('DRY RUN, CANDIDATES FOR QUEUE: ' + x)
else:
place_file_in_queue(x)
# %%
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="Watch and Fix Permissions And Push to Kafka Queue"
)
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
parser.add_argument("--dry-run", action='store_true', help='Dry Run')
parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')
args, _ = parser.parse_known_args()
paths = args.paths
force = args.force
dry_run = args.dry_run
if len(paths) == 0:
paths = default_dir_watch
add_files_to_queue(paths, dry_run = dry_run, force=force)
vision_v3/00_watch_for_new_videos/watch_and_fix_permissions.py → vision_v3/00_watch_for_new_videos/00_watch_and_fix_permissions.py
+12 -14
View File
@@ -8,36 +8,34 @@ import logging
from watchdog.observers.polling import PollingObserver
from watchdog.events import FileSystemEventHandler
from threading import Thread
from CommonCode import wq
from CommonCode.util import is_ipython
from common_code.util import is_ipython
import queue
from CommonCode.settings import get_logger, LogColorize
from common_code.settings import get_logger, LogColorize
import argparse
from common_code import kwq
import subprocess
from collections import deque
publish = kwq.producer.send
topic = kwq.TOPICS.exit_00_videos_to_score_detection
pfm = LogColorize.watch_and_fix_permissions
if not ('__file__' in vars() or '__file__' in globals()):
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
__file__ = '/home/thebears/Source/pipelines/vision_v3/00_watch_for_new_videos/00_watch_and_fix_permissions.py'
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir)
from common import settings
from common_code import settings
default_dir_watch = settings.dir_watch
tasks = queue.Queue()
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
logger.info(pfm(f"Starting file watcher @ {__file__}"))
try:
from CommonCode import kwq
publish = kwq.producer.send
topic = kwq.TOPICS.exit_00_videos_to_score_detection
except Exception as e:
logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
def decide_to_put_in_queue(file_name):
disqualifiers = list()
@@ -112,7 +110,7 @@ def process_file(filef):
if decide_to_put_in_queue(filef):
logging.info(f"QUEUE_PUT: {pfm(filef)}")
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
try:
future = publish(topic, key=filef, value={"filepath": filef})
except Exception as e:
vision_v3/00_watch_for_new_videos/__pycache__/load_missed_lib.cpython-312.pyc
BIN
View File
Binary file not shown.
vision_v3/00_watch_for_new_videos/load_missed_leopards34.py
+20
View File
@@ -0,0 +1,20 @@
from load_missed_lib import place_file_in_queue
# %%
import os
rrt = ['/srv/ftp_tcc/', '/mnt/hdd_24tb_1/videos/ftp']
dir_scan = ['leopards3','leopards4']
rt_scans = list()
for r1 in rrt:
for d1 in dir_scan:
rt_scans.append(os.path.join(r1, d1))
# %%
for rt_s in rt_scans:
for root, dirs, files in os.walk(rt_s):
for x in files:
print(x)
vision_v3/00_watch_for_new_videos/load_missed.py → vision_v3/00_watch_for_new_videos/load_missed_lib.py
+6 -7
View File
@@ -10,9 +10,9 @@ import logging
from watchdog.observers.polling import PollingObserver
from watchdog.events import FileSystemEventHandler
from threading import Thread
from CommonCode import wq
from CommonCode import kwq
from CommonCode.settings import get_logger, LogColorize
from common_code import kwq
from common_code.settings import get_logger, LogColorize
import argparse
if not ('__file__' in vars() or '__file__' in globals()):
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
@@ -20,7 +20,7 @@ if not ('__file__' in vars() or '__file__' in globals()):
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir)
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_load_missed', stdout=True, systemd=False, level=logging.INFO)
from common import settings
from common_code import settings
#default_dir_watch = settings.dir_watch_missed
default_dir_watch = settings.dir_watch
@@ -30,8 +30,7 @@ topic = kwq.TOPICS.exit_00_videos_to_score_detection
necessary_extensions = [
'.oclip_embeds.npz',
'.json',
'.db_has_sl2_embeds_part' ]
'.detection.npz']
def decide_to_put_in_queue(file_name):
if os.path.getsize(file_name) < 1*1000*1000:
@@ -71,7 +70,7 @@ def place_file_in_queue(filef):
logging.info(f"QUEUE_PUT: {filef}")
publish(topic, key=filef, value={"filepath": filef})
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
def add_files_to_queue(paths, dry_run = False):
vision_v3/00_watch_for_new_videos/test.py
-11
View File
@@ -1,11 +0,0 @@
from CommonCode import kwq
publish = kwq.producer.send
topic = kwq.TOPICS.videos_to_score_detection
cons = kwq.create_consumer()
# %%
cons.subscribe([topic])
# %%
if True:
out = cons.poll()
vision_v3/00_watch_for_new_videos/watch_and_fix_permissions_old.py
-162
View File
@@ -1,162 +0,0 @@
import os
import subprocess as sp
import time
import psutil
import stat
import sys
import logging
from watchdog.observers.polling import PollingObserver
from watchdog.events import FileSystemEventHandler
from threading import Thread
from CommonCode import wq
from CommonCode.settings import get_logger, LogColorize
import argparse
pfm = LogColorize.watch_and_fix_permissions
if not ('__file__' in vars() or '__file__' in globals()):
__file__ = '/home/thebears/Source/pipelines/00_watch_for_new_videos/watch_and_fix_permissions.py'
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(parent_dir)
from common import settings
default_dir_watch = settings.dir_watch
# %%
logger = get_logger(__name__,'/var/log/ml_vision_logs/00_watch_and_fix_permissions', stdout=True, systemd=False, level = logging.INFO)
logger.info(pfm(f"Starting watch_and_fix_permissions.py"))
try:
from CommonCode import kwq
publish = kwq.producer.send
topic = kwq.TOPICS.exit_00_videos_to_score_detection
except Exception as e:
logger.error("ERROR WHEN IMPORTING KAFKA:"+ str(e))
def decide_to_put_in_queue(file_name):
disqualifiers = list()
disqualifiers.append("_reduced" in file_name)
disqualifiers.append("_trimmed" in file_name)
disqualified = any(disqualifiers)
if file_name.endswith(".mp4") and not disqualified:
return True
else:
return False
class logic_execute(Thread):
def __init__(self, event):
logging.debug(f"EVENT: {event}")
Thread.__init__(self)
self.event = event
def run(self):
event = self.event
if isinstance(event, str):
src_path = event
else:
src_path = event.src_path
filef = os.path.abspath(os.path.join(os.getcwd(), src_path))
logging.debug(f"GOT_FILE: {pfm(filef)}")
broke_out = False
for i in range(120):
files = list()
for proc in psutil.process_iter():
try:
if "ftp" in proc.name():
files.append(proc.open_files())
except Exception:
pass
f = list()
for x in files:
f.extend(x)
if any([filef == e.path for e in f]):
pass
else:
if decide_to_put_in_queue(filef):
dirname = os.path.dirname(filef)
foldmode = stat.S_IMODE(os.stat(dirname).st_mode)
if oct(foldmode) != "0o777":
os.system(f"sudo chmod 777 {dirname}")
logging.info(f"SETTING_PERMISSIONS: {dirname}")
broke_out = True
break
time.sleep(1)
if not broke_out:
logging.info(f"TIMED_OUT: {filef}")
chand = {"root": os.path.dirname(filef), "name": os.path.basename(filef)}
try:
os.chmod(filef, 0o777)
except Exception as e:
logging.error(e)
if decide_to_put_in_queue(filef):
logging.info(f"QUEUE_PUT: {pfm(filef)}")
wq.publish(wq.TOPICS.ml_vision_to_score, filef)
try:
future = publish(topic, key=filef, value={"filepath": filef})
except Exception as e:
logging.error("KAFKA ERROR " + str(e))
else:
logging.info(f"QUEUE_DO_NOT_PUT: {filef}")
class Handler(FileSystemEventHandler):
def on_created(self, event):
try:
self.handle_it(event)
except Exception as e:
logging.error(e)
def handle_it(self, event):
if event.is_directory:
return
lclass = logic_execute(event)
lclass.start()
def observe_path(path):
event_handler = Handler()
observer = PollingObserver()
logger.info(f"Monitoring {path}")
observer.schedule(event_handler, path, recursive=True)
observer.start()
observer.is_alive()
return observer
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="Watch and Fix Permissions And Push to Kafka Queue"
)
parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
args, _ = parser.parse_known_args()
paths = args.paths
if len(paths) == 0:
paths = default_dir_watch
observers = [observe_path(path) for path in paths]
try:
while True:
time.sleep(1)
except Exception as e:
logging.error(str(e))
for observer in observers:
observer.stop()
observer.join()