55 lines
1.6 KiB
Python
55 lines
1.6 KiB
Python
#!/usr/bin/env python
|
|
from enum import Enum
|
|
import json
|
|
import sys
|
|
from kafka import KafkaProducer, KafkaConsumer
|
|
|
|
|
|
bootstrap_server = ["192.168.1.242:19092"]
|
|
|
|
|
|
class TOPICS():
|
|
videos_to_score_detection = "videos_to_score_detection"
|
|
videos_scored_detection="videos_scored_detection"
|
|
videos_with_nuggets="videos_with_nuggets"
|
|
videos_no_json="videos_no_json"
|
|
videos_without_nuggets="videos_without_nuggets"
|
|
videos_embedding_in_db="videos_embed_in_db"
|
|
videos_embedding_in_db_fail = "videos_embed_in_db_fail"
|
|
|
|
|
|
serializer = lambda v: json.dumps(v).encode("utf-8")
|
|
deserializer = json.loads
|
|
|
|
producer = KafkaProducer(
|
|
bootstrap_servers=bootstrap_server,
|
|
key_serializer=serializer,
|
|
value_serializer=serializer,
|
|
request_timeout_ms=15000, # 15s (keep small)
|
|
max_block_ms=10000, # 10s max blocking
|
|
metadata_max_age_ms=300000,
|
|
retry_backoff_ms=100,
|
|
linger_ms=5,
|
|
retries=3,
|
|
security_protocol='SASL_PLAINTEXT', # change to SASL_SSL if TLS is enabled
|
|
sasl_mechanism='SCRAM-SHA-256',
|
|
sasl_plain_username='superuser',
|
|
sasl_plain_password='marybear'
|
|
)
|
|
|
|
|
|
def create_consumer(group_id = None, client_id = None):
|
|
return KafkaConsumer(
|
|
bootstrap_servers=bootstrap_server,
|
|
key_deserializer=deserializer,
|
|
value_deserializer=deserializer,
|
|
enable_auto_commit = False,
|
|
group_id=group_id,
|
|
client_id = client_id,
|
|
auto_offset_reset = 'earliest',
|
|
security_protocol='SASL_PLAINTEXT', # change to SASL_SSL if TLS is enabled
|
|
sasl_mechanism='SCRAM-SHA-256',
|
|
sasl_plain_username='superuser',
|
|
sasl_plain_password='marybear'
|
|
)
|