from common_code.vector_utils import upload_to_doris
from tqdm import tqdm
import argparse
import os

def add_files_to_queue(paths, dry_run = False, force = False):
    queued = set()
    for rt in paths:
        for root, dirs, files in os.walk(rt):
            for f in files:
                new_path = os.path.join(root, f)
                if new_path.endswith('.oclip_embeds.npz'):
#                if decide_to_put_in_queue(new_path, force = force):
                    queued.add(new_path)

    for x in tqdm(queued):
        try:
            upload_to_doris(x)
        except Exception as e:
            print('Failed: '+str(x)+'with '+str(e))

        



if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="Watch and Fix Permissions And Push to Kafka Queue"
    )

    parser.add_argument("paths", nargs="*", help="Paths to monitor", default=())
    parser.add_argument("--dry-run", action='store_true', help='Dry Run')
    parser.add_argument("--force", action='store_true', help='Force adding (do not check if already done)')    
    args, _ = parser.parse_known_args()
    paths = args.paths
    force = args.force
    dry_run = args.dry_run

    if len(paths) == 0:
        paths = default_dir_watch

    add_files_to_queue(paths, dry_run = dry_run, force=force)

