#!/usr/bin/env python from enum import Enum import json import sys from kafka import KafkaProducer, KafkaConsumer bootstrap_server = ["192.168.1.242:19092"] class TOPICS(): videos_to_score_detection = "videos_to_score_detection" videos_scored_detection="videos_scored_detection" videos_with_nuggets="videos_with_nuggets" videos_no_json="videos_no_json" videos_without_nuggets="videos_without_nuggets" videos_embedding_in_db="videos_embed_in_db" videos_embedding_in_db_fail = "videos_embed_in_db_fail" serializer = lambda v: json.dumps(v).encode("utf-8") deserializer = json.loads producer = KafkaProducer( bootstrap_servers=bootstrap_server, key_serializer=serializer, value_serializer=serializer, request_timeout_ms=15000, # 15s (keep small) max_block_ms=10000, # 10s max blocking metadata_max_age_ms=300000, retry_backoff_ms=100, linger_ms=5, retries=3, security_protocol='SASL_PLAINTEXT', # change to SASL_SSL if TLS is enabled sasl_mechanism='SCRAM-SHA-256', sasl_plain_username='superuser', sasl_plain_password='marybear' ) def create_consumer(group_id = None, client_id = None): return KafkaConsumer( bootstrap_servers=bootstrap_server, key_deserializer=deserializer, value_deserializer=deserializer, enable_auto_commit = False, group_id=group_id, client_id = client_id, auto_offset_reset = 'earliest', security_protocol='SASL_PLAINTEXT', # change to SASL_SSL if TLS is enabled sasl_mechanism='SCRAM-SHA-256', sasl_plain_username='superuser', sasl_plain_password='marybear' )