YACWC

kwq.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+from enum import Enum
+import json
+import sys
+from kafka import KafkaProducer, KafkaConsumer
+
+
+bootstrap_server = ["192.168.1.242:19092"]
+
+
+class TOPICS():
+    videos_to_score_detection = "videos_to_score_detection"
+    videos_scored_detection="videos_scored_detection"
+    videos_with_nuggets="videos_with_nuggets"
+    videos_no_json="videos_no_json"
+    videos_without_nuggets="videos_without_nuggets"
+    videos_embedding_in_db="videos_embed_in_db"
+    videos_embedding_in_db_fail = "videos_embed_in_db_fail"
+
+
+serializer = lambda v: json.dumps(v).encode("utf-8")
+deserializer = json.loads
+
+producer = KafkaProducer(
+    bootstrap_servers=bootstrap_server,
+    key_serializer=serializer,
+    value_serializer=serializer,
+    request_timeout_ms=15000, # 15s (keep small)
+    max_block_ms=10000,       # 10s max blocking
+    metadata_max_age_ms=300000,
+    retry_backoff_ms=100,
+    linger_ms=5,
+    retries=3,
+    security_protocol='SASL_PLAINTEXT',  # change to SASL_SSL if TLS is enabled
+    sasl_mechanism='SCRAM-SHA-256',
+    sasl_plain_username='superuser',
+    sasl_plain_password='marybear'
+)
+
+
+def create_consumer(group_id = None, client_id = None):
+    return KafkaConsumer(
+        bootstrap_servers=bootstrap_server,
+        key_deserializer=deserializer,
+        value_deserializer=deserializer,
+        enable_auto_commit = False,
+        group_id=group_id,
+        client_id = client_id,
+        auto_offset_reset = 'earliest',
+        security_protocol='SASL_PLAINTEXT',  # change to SASL_SSL if TLS is enabled
+        sasl_mechanism='SCRAM-SHA-256',
+        sasl_plain_username='superuser',
+        sasl_plain_password='marybear'        
+    )