543 lines
17 KiB
Python
Executable File
543 lines
17 KiB
Python
Executable File
import sys
|
|
|
|
sys.path.append("/opt/nvidia/deepstream/deepstream/sources/deepstream_python_apps/apps")
|
|
import os
|
|
|
|
import gi
|
|
|
|
gi.require_version("Gst", "1.0")
|
|
import argparse
|
|
import ctypes
|
|
from functools import partial
|
|
|
|
import numpy as np
|
|
import pyds
|
|
from common.bus_call import bus_call
|
|
from common.platform_info import PlatformInfo
|
|
from CommonCode.settings import LogColorize, get_logger
|
|
from gi.repository import GLib, Gst
|
|
|
|
pfm = LogColorize.watch_and_fix_permissions
|
|
|
|
logger = get_logger(
|
|
__name__,
|
|
"/var/log/ml_vision_logs/00_watch_and_fix_permissions",
|
|
stdout=True,
|
|
systemd=False,
|
|
)
|
|
Gst.debug_set_default_threshold(Gst.DebugLevel.ERROR)
|
|
os.environ.pop("DISPLAY", ":0")
|
|
target_width_detect = 1280
|
|
target_height_detect = 720
|
|
os.environ["GST_DEBUG_DUMP_DOT_DIR"] = "/tmp"
|
|
os.putenv("GST_DEBUG_DUMP_DIR_DIR", "/tmp")
|
|
|
|
target_width_embed = 512
|
|
target_height_embed = 512
|
|
|
|
MUXER_BATCH_TIMEOUT_USEC = 1000000
|
|
|
|
|
|
def print_pipeline_structure(pipeline):
|
|
"""
|
|
Recursively prints elements in the pipeline and their properties.
|
|
"""
|
|
if not isinstance(pipeline, Gst.Pipeline):
|
|
print("Not a valid GStreamer pipeline.")
|
|
return
|
|
|
|
def _print_element_properties(element, indent=0):
|
|
spaces = " " * indent
|
|
print(
|
|
spaces
|
|
+ f"Element: {element.get_name()} (Type: {element.get_factory().get_name()})"
|
|
)
|
|
|
|
# Print its properties
|
|
for prop in element.list_properties():
|
|
try:
|
|
val = element.get_property(prop.name)
|
|
if val != prop.default_value: # Display only non-default properties
|
|
print(spaces + f" - {prop.name}: {val}")
|
|
except:
|
|
pass
|
|
|
|
def _print_pipeline_structure(element, indent=0):
|
|
spaces = " " * indent
|
|
children = element.children if hasattr(element, "children") else []
|
|
|
|
if len(children) > 0:
|
|
print(spaces + f"[{element.get_name()}]")
|
|
for child in children:
|
|
_print_pipeline_structure(child, indent + 2)
|
|
else:
|
|
_print_element_properties(element, indent)
|
|
|
|
print("\nPipeline Structure:")
|
|
print("===================")
|
|
_print_pipeline_structure(pipeline)
|
|
print("===================\n")
|
|
|
|
|
|
def get_detailed_pipeline_string(pipeline):
|
|
"""Generate a more detailed pipeline string with properties"""
|
|
if not isinstance(pipeline, Gst.Pipeline):
|
|
return None
|
|
|
|
def get_element_string(element):
|
|
# Get element factory name
|
|
factory = element.get_factory()
|
|
if factory:
|
|
element_str = factory.get_name()
|
|
else:
|
|
element_str = element.get_name()
|
|
|
|
# Add properties
|
|
props = []
|
|
for prop in element.list_properties():
|
|
# Skip some properties that are typically not set in command line
|
|
if prop.name in ("name", "parent"):
|
|
continue
|
|
|
|
try:
|
|
val = element.get_property(prop.name)
|
|
if val is not None and val != prop.default_value:
|
|
# Format value appropriately based on type
|
|
if isinstance(val, str):
|
|
props.append(f'{prop.name}="{val}"')
|
|
elif isinstance(val, bool):
|
|
props.append(f"{prop.name}={str(val).lower()}")
|
|
else:
|
|
props.append(f"{prop.name}={val}")
|
|
except:
|
|
# Skip properties that can't be read
|
|
pass
|
|
|
|
if props:
|
|
element_str += " " + " ".join(props)
|
|
|
|
return element_str
|
|
|
|
result = []
|
|
|
|
# Simple approach - just gets top-level elements
|
|
iterator = pipeline.iterate_elements()
|
|
while True:
|
|
ret, element = iterator.next()
|
|
if ret != Gst.IteratorResult.OK:
|
|
break
|
|
result.append(get_element_string(element))
|
|
|
|
return " ! ".join(result)
|
|
|
|
|
|
def embed_results_probe(pad, info, u_data, list_add, frame_num=0):
|
|
gst_buffer = info.get_buffer()
|
|
print("HEY I AM PROBING EMBEDDINGS")
|
|
if not gst_buffer:
|
|
print("Unable to get GstBuffer ")
|
|
return
|
|
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
|
|
l_frame = batch_meta.frame_meta_list
|
|
|
|
while l_frame is not None:
|
|
try:
|
|
# Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
|
|
# The casting also keeps ownership of the underlying memory
|
|
# in the C code, so the Python garbage collector will leave
|
|
# it alone.
|
|
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
|
|
except StopIteration:
|
|
break
|
|
|
|
frame_number = frame_meta.frame_num
|
|
|
|
l_user = frame_meta.frame_user_meta_list
|
|
while l_user is not None:
|
|
try:
|
|
# Note that l_user.data needs a cast to pyds.NvDsUserMeta
|
|
# The casting also keeps ownership of the underlying memory
|
|
# in the C code, so the Python garbage collector will leave
|
|
# it alone.
|
|
user_meta = pyds.NvDsUserMeta.cast(l_user.data)
|
|
except StopIteration:
|
|
break
|
|
|
|
if (
|
|
user_meta.base_meta.meta_type
|
|
!= pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META
|
|
):
|
|
continue
|
|
|
|
tensor_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
|
|
|
|
# Boxes in the tensor meta should be in network resolution which is
|
|
# found in tensor_meta.network_info. Use this info to scale boxes to
|
|
# the input frame resolution.
|
|
layers_info = []
|
|
if True:
|
|
for i in range(tensor_meta.num_output_layers):
|
|
layer = pyds.get_nvds_LayerInfo(tensor_meta, i)
|
|
if layer.layerName == "embedding":
|
|
|
|
ptr = ctypes.cast(
|
|
pyds.get_ptr(layer.buffer), ctypes.POINTER(ctypes.c_float)
|
|
)
|
|
num_elements = layer.inferDims.numElements
|
|
v = list(np.ctypeslib.as_array(ptr, shape=(num_elements,)))
|
|
v = [float(x) for x in v]
|
|
|
|
list_add.append({"frame_number": frame_number, "vector": v})
|
|
|
|
try:
|
|
l_user = l_user.next
|
|
except StopIteration:
|
|
break
|
|
|
|
try:
|
|
# indicate inference is performed on the frame
|
|
frame_meta.bInferDone = True
|
|
l_frame = l_frame.next
|
|
except StopIteration:
|
|
break
|
|
|
|
return Gst.PadProbeReturn.OK
|
|
|
|
|
|
def detector_results_probe(pad, info, u_data, list_add, frame_num=0):
|
|
frame_number = 0
|
|
num_rects = 0
|
|
got_fps = False
|
|
print("HEY I AM PROBING DETECTIONS")
|
|
gst_buffer = info.get_buffer()
|
|
if not gst_buffer:
|
|
print("Unable to get GstBuffer ")
|
|
return
|
|
|
|
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
|
|
l_frame = batch_meta.frame_meta_list
|
|
|
|
while l_frame is not None:
|
|
try:
|
|
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
|
|
except StopIteration:
|
|
break
|
|
|
|
frame_number = frame_meta.frame_num
|
|
l_obj = frame_meta.obj_meta_list
|
|
num_rects = frame_meta.num_obj_meta
|
|
|
|
l_user = frame_meta.frame_user_meta_list
|
|
|
|
while l_obj is not None:
|
|
try:
|
|
# Casting l_obj.data to pyds.NvDsObjectMeta
|
|
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
|
|
except StopIteration:
|
|
break
|
|
|
|
# param_extract = ['left','top','width','height']
|
|
# strc = ''
|
|
# for param in param_extract:
|
|
# strc+=str(getattr(obj_meta.rect_params, param))
|
|
# strc+=' '
|
|
|
|
# target_width
|
|
# target_height
|
|
score = obj_meta.confidence
|
|
label = obj_meta.obj_label
|
|
left = obj_meta.rect_params.left
|
|
top = obj_meta.rect_params.top
|
|
width = obj_meta.rect_params.width
|
|
height = obj_meta.rect_params.height
|
|
frame_number = frame_number
|
|
class_id = obj_meta.class_id
|
|
|
|
d_add = {
|
|
"score": score,
|
|
"label": label,
|
|
"left": left,
|
|
"top": top,
|
|
"width": width,
|
|
"height": height,
|
|
"frame_number": frame_number,
|
|
"class_id": class_id,
|
|
}
|
|
list_add.append(d_add)
|
|
print(frame_number, label, score)
|
|
if frame_number % 100 == 0:
|
|
str_pr = "FRAME_PROGRESS: " + pfm(
|
|
str(frame_number) + "/" + str(frame_num)
|
|
)
|
|
logger.info(str_pr)
|
|
|
|
try:
|
|
l_obj = l_obj.next
|
|
except StopIteration:
|
|
break
|
|
|
|
# Update frame rate through this probe
|
|
stream_index = "stream{0}".format(frame_meta.pad_index)
|
|
|
|
try:
|
|
l_frame = l_frame.next
|
|
except StopIteration:
|
|
break
|
|
|
|
return Gst.PadProbeReturn.OK
|
|
|
|
|
|
def cb_newpad(decodebin, decoder_src_pad, data):
|
|
caps = decoder_src_pad.get_current_caps()
|
|
if not caps:
|
|
caps = decoder_src_pad.query_caps()
|
|
gststruct = caps.get_structure(0)
|
|
gstname = gststruct.get_name()
|
|
source_bin = data
|
|
features = caps.get_features(0)
|
|
|
|
# Need to check if the pad created by the decodebin is for video and not
|
|
# audio.
|
|
print("gstname=", gstname)
|
|
if gstname.find("video") != -1:
|
|
# Link the decodebin pad only if decodebin has picked nvidia
|
|
# decoder plugin nvdec_*. We do this by checking if the pad caps contain
|
|
# NVMM memory features.
|
|
print("features=", features)
|
|
if features.contains("memory:NVMM"):
|
|
# Get the source bin ghost pad
|
|
bin_ghost_pad = source_bin.get_static_pad("src")
|
|
if not bin_ghost_pad.set_target(decoder_src_pad):
|
|
sys.stderr.write(
|
|
"Failed to link decoder src pad to source bin ghost pad\n"
|
|
)
|
|
else:
|
|
sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n")
|
|
|
|
|
|
def decodebin_child_added(child_proxy, Object, name, user_data):
|
|
print("Decodebin child added:", name, "\n")
|
|
if name.find("decodebin") != -1:
|
|
Object.connect("child-added", decodebin_child_added, user_data)
|
|
|
|
if "source" in name:
|
|
source_element = child_proxy.get_by_name("source")
|
|
if source_element.find_property("drop-on-latency") != None:
|
|
Object.set_property("drop-on-latency", True)
|
|
|
|
|
|
def create_source_bin(uri):
|
|
|
|
bin_name = "source-bin-any-format"
|
|
|
|
nbin = Gst.Bin.new(bin_name)
|
|
if not nbin:
|
|
sys.stderr.write(" Unable to create source bin \n")
|
|
|
|
uri_decode_bin = Gst.ElementFactory.make("uridecodebin", "uri-decode-bin")
|
|
if not uri_decode_bin:
|
|
sys.stderr.write(" Unable to create uri decode bin \n")
|
|
|
|
uri_decode_bin.set_property("uri", uri)
|
|
|
|
uri_decode_bin.connect("pad-added", cb_newpad, nbin)
|
|
uri_decode_bin.connect("child-added", decodebin_child_added, nbin)
|
|
|
|
Gst.Bin.add(nbin, uri_decode_bin)
|
|
bin_pad = nbin.add_pad(Gst.GhostPad.new_no_target("src", Gst.PadDirection.SRC))
|
|
if not bin_pad:
|
|
sys.stderr.write(" Failed to add ghost pad in source bin \n")
|
|
return None
|
|
return nbin
|
|
|
|
|
|
# def run_inference(file_path):
|
|
if True:
|
|
file_path = "/home/thebears/local/source/short.mp4"
|
|
os.environ.pop("DISPLAY", None)
|
|
if not file_path.startswith("file://"):
|
|
file_path = "file://" + file_path
|
|
|
|
platform_info = PlatformInfo()
|
|
Gst.init(None)
|
|
|
|
pipeline = Gst.Pipeline()
|
|
|
|
source_file = create_source_bin(file_path)
|
|
|
|
tee = Gst.ElementFactory.make("tee", "nvsink-tee")
|
|
|
|
# DETECT
|
|
queue_detect = Gst.ElementFactory.make("queue", "nvtee-detect")
|
|
|
|
streammux_detect = Gst.ElementFactory.make("nvstreammux", "Stream-muxer-detector")
|
|
streammux_detect.set_property("width", target_width_detect)
|
|
streammux_detect.set_property("height", target_height_detect)
|
|
streammux_detect.set_property("batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC)
|
|
streammux_detect.set_property("enable-padding", 1)
|
|
streammux_detect.set_property("batch-size", 4)
|
|
|
|
nugget_detector = Gst.ElementFactory.make("nvinfer", "primary-inference")
|
|
nugget_detector.set_property(
|
|
"config-file-path",
|
|
"/home/thebears/DeepStream-Yolo/config_infer_primary_yoloV7.txt",
|
|
)
|
|
|
|
fakesink_detect = Gst.ElementFactory.make("fakesink", "fakesink")
|
|
fakesink_detect.set_property("enable-last-sample", 0)
|
|
fakesink_detect.set_property("sync", 0)
|
|
|
|
# EMBED
|
|
|
|
queue_embed = Gst.ElementFactory.make("queue", "nvtee-que-embed")
|
|
|
|
streammux_embed = Gst.ElementFactory.make("nvstreammux", "Stream-muxer-embed")
|
|
streammux_embed.set_property("width", target_width_embed)
|
|
streammux_embed.set_property("height", target_height_embed)
|
|
streammux_embed.set_property("batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC)
|
|
streammux_embed.set_property("enable-padding", 1)
|
|
streammux_embed.set_property("batch-size", 4)
|
|
|
|
nugget_embed = Gst.ElementFactory.make("nvinfer", "primary-inference")
|
|
nugget_embed.set_property(
|
|
"config-file-path", "/home/thebears/DeepStream-Yolo/embedder.txt"
|
|
)
|
|
# nugget_embed.set_property('config-file-path', "/home/thebears/DeepStream-Yolo/config_infer_primary_yoloV7.txt")
|
|
|
|
fakesink_embed = Gst.ElementFactory.make("fakesink", "fakesink2")
|
|
fakesink_embed.set_property("enable-last-sample", 0)
|
|
fakesink_embed.set_property("sync", 0)
|
|
|
|
# LINKING
|
|
|
|
# Ensure NVMM caps with a capsfilter
|
|
# capsfilter = Gst.ElementFactory.make("capsfilter", "capsfilter")
|
|
# capsfilter.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM), format=NV12"))
|
|
# pipeline.add(capsfilter)
|
|
|
|
pipeline.add(source_file)
|
|
pipeline.add(tee)
|
|
|
|
nvvidconv = Gst.ElementFactory.make("nvvidconv", "nvvidconv")
|
|
pipeline.add(nvvidconv)
|
|
|
|
source_file.link(nvvidconv)
|
|
# nvvidconv.link(capsfilter)
|
|
# capsfilter.link(tee)
|
|
nvvidconv.link(tee)
|
|
|
|
if True:
|
|
pipeline.add(queue_detect)
|
|
pipeline.add(streammux_detect)
|
|
pipeline.add(nugget_detector)
|
|
pipeline.add(fakesink_detect)
|
|
|
|
tee.get_request_pad("src_%u").link(queue_detect.get_static_pad("sink"))
|
|
queue_detect.get_static_pad("src").link(
|
|
streammux_detect.get_request_pad("sink_0")
|
|
)
|
|
streammux_detect.link(nugget_detector)
|
|
nugget_detector.link(fakesink_detect)
|
|
|
|
os.environ["GST_DEBUG_DUMP_DOT_DIR"] = "/tmp"
|
|
os.putenv("GST_DEBUG_DUMP_DIR_DIR", "/tmp")
|
|
|
|
if False:
|
|
|
|
pipeline.add(queue_embed)
|
|
pipeline.add(streammux_embed)
|
|
pipeline.add(nugget_embed)
|
|
pipeline.add(fakesink_embed)
|
|
|
|
tee.get_request_pad("src_%u").link(queue_embed.get_static_pad("sink"))
|
|
queue_embed.get_static_pad("src").link(
|
|
streammux_embed.get_request_pad("sink_0")
|
|
)
|
|
streammux_embed.link(nugget_embed)
|
|
nugget_embed.link(fakesink_embed)
|
|
|
|
Gst.debug_bin_to_dot_file(pipeline, Gst.DebugGraphDetails.ALL, "pipeline")
|
|
print_pipeline_structure(pipeline)
|
|
cmd = f"/usr/bin/ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 {file_path}" # /srv/ftp/railing/2025/02/28/railing_00_20250228115800.mp4
|
|
try:
|
|
frames = int(os.popen(cmd).read().strip())
|
|
except:
|
|
frames = 0
|
|
|
|
logger.info(f"TOTAL_FRAMES: {frames}")
|
|
|
|
embed_list = list()
|
|
|
|
Gst.debug_bin_to_dot_file(pipeline, Gst.DebugGraphDetails.ALL, "pipeline_structure")
|
|
embed_results = partial(embed_results_probe, list_add=embed_list, frame_num=frames)
|
|
nugget_embed.get_static_pad("src").add_probe(
|
|
Gst.PadProbeType.BUFFER, embed_results, 0
|
|
)
|
|
|
|
Gst.debug_bin_to_dot_file(
|
|
pipeline,
|
|
Gst.DebugGraphDetails.ALL,
|
|
"/home/thebears/local/source/pipeline_structure",
|
|
)
|
|
|
|
detector_list = list()
|
|
|
|
detector_results = partial(
|
|
detector_results_probe, list_add=detector_list, frame_num=frames
|
|
)
|
|
nugget_detector.get_static_pad("src").add_probe(
|
|
Gst.PadProbeType.BUFFER, detector_results, 0
|
|
)
|
|
|
|
print("AFTER SETTING STATIC PADS")
|
|
|
|
def get_pipeline_string(pipeline):
|
|
if not isinstance(pipeline, Gst.Pipeline):
|
|
return None
|
|
|
|
elements = []
|
|
iterator = pipeline.iterate_elements()
|
|
while True:
|
|
result, element = iterator.next()
|
|
if result != Gst.IteratorResult.OK:
|
|
break
|
|
elements.append(element.get_name())
|
|
|
|
return " ! ".join(elements)
|
|
|
|
Gst.debug_bin_to_dot_file(pipeline, Gst.DebugGraphDetails.ALL, "pipeline_structure")
|
|
|
|
# create an event loop and feed gstreamer bus mesages to it
|
|
loop = GLib.MainLoop()
|
|
bus = pipeline.get_bus()
|
|
bus.add_signal_watch()
|
|
bus.connect("message", bus_call, loop)
|
|
|
|
# start play back and listen to events
|
|
print("Starting pipeline \n")
|
|
pipeline.set_state(Gst.State.PLAYING)
|
|
try:
|
|
loop.run()
|
|
except:
|
|
pass
|
|
# cleanup
|
|
pipeline.set_state(Gst.State.NULL)
|
|
# return detector_list, embed_list
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cpath = sys.argv[1]
|
|
if cpath.endswith("-i"):
|
|
cpath = "/home/thebears/local/source/short.mp4"
|
|
|
|
if not cpath.startswith("file"):
|
|
cpath = os.path.abspath(cpath)
|
|
|
|
out = run_inference(cpath)
|
|
|
|
import json
|
|
|
|
with open("dump.json", "w") as ff:
|
|
json.dump([out[0], out[1]], ff)
|
|
sys.exit()
|