YACWC

wq.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+import redis, pickle
+from enum import Enum
+import json
+
+class TOPICS(Enum):
+    ml_vision_to_score = 'ml_vision_to_score'
+    ml_vision_to_score_ext = 'ml_vision_to_score_ext'
+    ml_vision_objdet_failed = 'ml_vision_objdet_failed'
+    ml_vision_objdet_success = 'ml_vision_objdet_success'
+    ml_vision_objdet_skipped = 'ml_vision_objdet_skipped'
+    ml_vision_objdet_results_db_success = 'ml_vision_objdet_db_upload_success'
+    ml_vision_objdet_results_db_failed = 'ml_vision_objdet_db_upload_failed'
+    ml_vision_objdet_results_pg_success = 'ml_vision_objdet_pg_upload_success'
+    ml_vision_objdet_results_pg_failed = 'ml_vision_objdet_pg_upload_failed'
+    ml_vision_objdet_results_purge_success = 'ml_vision_objdet_purge_success'    
+    ml_vision_objdet_results_purge_failed = 'ml_vision_objdet_purge_failed'
+    ml_vision_objdet_results_purge_skipped = 'ml_vision_objdet_purge_skipped'
+    ml_vision_videos_modify_success = 'ml_vision_videos_modify_success'
+    ml_vision_videos_modify_failed = 'ml_vision_videos_modify_failed'
+    ml_vision_embedding_success = 'ml_vision_embedding_success'
+    ml_vision_embedding_fail = 'ml_vision_embedding_fail'
+    ml_vision_embedding_skipped = 'ml_vision_embedding_skipped'
+
+
+r = redis.StrictRedis()
+
+def publish(topic, message):
+    if isinstance(topic, TOPICS):
+        topic = topic.value
+
+    
+    if isinstance(message, str):
+        r.rpush(topic, message)
+    else:
+        r.rpush(topic, json.dumps(message))
+
+
+def parse_message(msg):
+    try:
+        return json.loads(msg)
+    except:
+        return msg.decode()
+        
+def consume(topic):
+    if isinstance(topic, TOPICS):
+        topic = topic.value
+
+    msg = r.rpop(topic)
+    if msg is None:
+        return None
+    else:
+        return parse_message(msg)
+        
+def peek(topic):
+    if isinstance(topic, TOPICS):
+        topic = topic.value
+
+    vals = r.lrange(topic,0, 0)
+    if len(vals) == 0:
+        return None
+    else:
+        return parse_message(vals[0])
+
+# from kafka import KafkaProducer, KafkaConsumer
+# import pickle
+
+# producer = KafkaProducer(bootstrap_servers=["192.168.1.242:9092"])
+# consumers = dict()
+# # %%
+
+# def publish(topic, message):
+#     producer.send(topic, value=pickle.dumps(message))
+#     producer.flush()
+
+
+# def get_consumer_by_topic(topic):
+#     if topic in consumers:
+#         return consumers[topic]
+
+#     csumer = KafkaConsumer(
+#         bootstrap_servers="localhost:9092",
+#         group_id="wq",
+#         auto_offset_reset="earliest",
+#         enable_auto_commit=True,
+#         consumer_timeout_ms=1000,
+#     )
+#     csumer.subscribe(topic)
+#     consumers[topic] = csumer
+
+
+# def get_a_message(topic):
+#     csumer = get_consumer_by_topic(topic)
+#     return csumer
+    
+