YACWC

SearchFrontend/search_ui/.prettierrc

@@ -1,4 +1,5 @@
 {
-  "tabWidth": 2,
+    "tabWidth": 4,
-  "useTabs": false
+    "useTabs": false,
+    "experimentalOperatorPosition": "start"
 }

SearchFrontend/search_ui/package-lock.json

@@ -1753,9 +1753,9 @@
       "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A=="
     },
     "error-ex": {
-      "version": "1.3.2",
+      "version": "1.3.4",
-      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz",
-      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+      "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==",
       "requires": {
         "is-arrayish": "^0.2.1"
       }
@@ -2561,6 +2561,11 @@
         "prop-types": "^15.5.4"
       }
     },
+    "react-table": {
+      "version": "7.8.0",
+      "resolved": "https://registry.npmjs.org/react-table/-/react-table-7.8.0.tgz",
+      "integrity": "sha512-hNaz4ygkZO4bESeFfnfOft73iBUj8K5oKi1EcSHPAibEydfsX2MyU6Z8KCr3mv3C9Kqqh71U+DhZkFvibbnPbA=="
+    },
     "react-transition-group": {
       "version": "4.4.5",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",

SearchFrontend/search_ui/package.json

@@ -29,6 +29,7 @@
     "react-dom": "^19.1.1",
     "react-flexbox-grid": "^2.1.2",
     "react-split-pane": "^0.1.92",
+    "react-table": "^7.8.0",
     "rsuite": "^5.83.3",
     "timelines-chart": "^2.14.2",
     "uplot": "^1.6.32",

SearchFrontend/search_ui/src/App.jsx

@@ -50,9 +50,7 @@ function App() {
         sliderValue !== lastSubmitted.sliderValue ||
         queryText !== lastSubmitted.queryText;
 
-  // Function to resubmit fetch
+    const streamComputeStatus = () => {
-  const handleResubmit = () => {
-    // Start streaming status updates
         fetch("api/return_status")
             .then((response) => {
                 const reader = response.body.getReader();
@@ -63,10 +61,8 @@ function App() {
                     reader.read().then(({ done, value }) => {
                         if (done) {
                             if (buffer) {
-                // console.log("Status:", buffer); // Log any remaining text
                             }
                             setStatusMessages([]);
-              // console.log("Status stream finished");
 
                             return;
                         }
@@ -79,9 +75,15 @@ function App() {
 
                         for (const line of lines) {
                             if (line.trim()) {
-                // console.log("Status:", line);
+                                let c_line = JSON.parse(line);
-                console.log(line)
+
-                setStatusMessages((msgs) => [...msgs, JSON.parse(line)]);
+                                if (c_line["task"] !== "DONE_QUIT") {
+                                  console.log(c_line);
+                                    setStatusMessages((msgs) => [
+                                        ...msgs,
+                                        c_line,
+                                    ]);
+                                }
                             }
                         }
 
@@ -93,10 +95,32 @@ function App() {
             .catch((error) => {
                 console.error("Error while streaming status:", error);
             });
+    };
+    // Function to resubmit fetch
+    const handleResubmit = (doTestMode = false) => {
+        console.log("startRange, endRange:", startRange, endRange);
+        console.log("test mode:", doTestMode);
+        let startRangeUse;
+        let endRangeUse;
+        if (doTestMode == true) {
+            startRangeUse = new Date(
+                new Date().getTime() - 2 * 24 * 60 * 60 * 1000
+            );
+            endRangeUse = new Date(
+                new Date().getTime() - 1 * 24 * 60 * 60 * 1000
+            );
+        } else {
+            startRangeUse = startRange;
+            endRangeUse = endRange;
+        }
+
+        console.log("Using date range:", startRangeUse, endRangeUse);
+        setStartRange(startRangeUse);
+        setEndRange(endRangeUse);
 
         const params = new URLSearchParams();
-    params.append("startRange", startRange.toISOString());
+        params.append("startRange", startRangeUse.toISOString());
-    params.append("endRange", endRange.toISOString());
+        params.append("endRange", endRangeUse.toISOString());
         params.append("threshold", 0.0);
         params.append("query", queryText);
         setDataResults({ videos: [], breaks: [] });
@@ -104,16 +128,17 @@ function App() {
         fetch("api/videos.json?" + params.toString())
             .then((res) => res.json())
             .then((data) => {
-        const max_value = Math.max(
+                streamComputeStatus();
-          ...data["videos"].map((vid) => vid["embed_scores"]["score"][1])
+                // Don't setDataResults here, since it's just {"status": ...}
-        );
+                pollForResult(); // Start polling for the real result
-        setSliderMax(max_value);
-        original_data.current = data;
-        window.original_data = original_data;
-        setDataResults(data);
             });
 
-    setLastSubmitted({ startRange, endRange, sliderValue, queryText });
+        setLastSubmitted({
+            startRangeUse,
+            endRangeUse,
+            sliderValue,
+            queryText,
+        });
     };
 
     function updateDataAndValue(newValue) {
@@ -181,10 +206,7 @@ function App() {
     // Memoize the timeline click handler
     const handleTimelineClick = useCallback(
         (path, timeoffset) => {
-      console.log("Timeline clicked:", path, timeoffset);
-
             if (playerRef.current && playerInstanceRef.current) {
-        console.log("Seeking video player to:", path, timeoffset);
                 playerInstanceRef.current.src({
                     src: "api/" + path,
                     type: "video/mp4",
@@ -198,15 +220,40 @@ function App() {
     );
 
     useEffect(() => {
-    const params = new URLSearchParams(window.location.search); // id=123
+        const params = new URLSearchParams(window.location.search);
 
-    if (params.get("test_mode") == "true") {
+        handleResubmit(params.get("test_mode") === "true");
-      setStartRange(new Date(new Date().getTime() - 2 * 24 * 60 * 60 * 1000));
-      setEndRange(new Date(new Date().getTime() - 1 * 24 * 60 * 60 * 1000));
-    }
-    handleResubmit();
     }, []);
 
+    // useEffect(() => {
+    //   if (startRange && endRange) {
+    //     handleResubmit();
+    //   }
+    // }, [startRange, endRange]);
+
+    function pollForResult() {
+        const poll = () => {
+            fetch("api/videos_result.json")
+                .then((res) => res.json())
+                .then((data) => {
+                    if (data.status === "processing") {
+                        setTimeout(poll, 250); // Try again in 1 second
+                    } else {
+                        const max_value = Math.max(
+                            ...data["videos"].map(
+                                (vid) => vid["embed_scores"]["score"][1]
+                            )
+                        );
+                        setSliderMax(max_value);
+
+                        original_data.current = data;
+                        setDataResults(data);
+                    }
+                });
+        };
+        poll();
+    }
+
     return (
         <div className="app-container">
             <div className="section-box-horiz">
@@ -237,7 +284,11 @@ function App() {
                 </div>
                 <div className="flex-group">
                     <label
-            style={{ marginLeft: "8px", marginRight: "8px", color: "#fff" }}
+                        style={{
+                            marginLeft: "8px",
+                            marginRight: "8px",
+                            color: "#fff",
+                        }}
                     >
                         Threshold:
                     </label>

SearchFrontend/search_ui/src/components/StatusDisplay.css

@@ -0,0 +1,90 @@
+/* Container */
+.table-container {
+  margin-top: 24px;
+  padding: 20px;
+  background: #23272f;
+  border-radius: 12px;
+  box-shadow: 0 2px 12px rgba(0,0,0,0.10);
+  overflow-x: auto;
+}
+
+/* Table */
+table {
+  width: 100%;
+  border-collapse: separate;
+  border-spacing: 0;
+  background: #23272f;
+  color: #e0e6ed;
+  font-family: 'Segoe UI', Arial, sans-serif;
+  font-size: 1rem;
+  box-shadow: 0 1px 4px rgba(0,0,0,0.06);
+}
+
+/* Table Header */
+th {
+  background: linear-gradient(90deg, #2c313c 80%, #23272f 100%);
+  color: #a9b7c6;
+  font-weight: 600;
+  padding: 12px 10px;
+  border-bottom: 2px solid #3a7afe;
+  text-align: left;
+  letter-spacing: 0.03em;
+  position: sticky;
+  top: 0;
+  z-index: 2;
+}
+
+/* Table Body */
+td {
+  padding: 10px 10px;
+  border-bottom: 1px solid #343a40;
+  background: #23272f;
+  color: #e0e6ed;
+  vertical-align: middle;
+  transition: background 0.2s;
+}
+
+/* Row hover */
+tr:hover td {
+  background: #2c313c;
+}
+
+/* First column highlight */
+td:first-child, th:first-child {
+  font-weight: 500;
+  color: #3a7afe;
+}
+
+/* Last row no border */
+tr:last-child td {
+  border-bottom: none;
+}
+
+/* Table column widths */
+table th:last-child,
+table td:last-child {
+  width: 30%;
+  min-width: 120px;
+  max-width: 400px;
+  /* Prevent shrinking below 30% on wide screens */
+}
+
+table th:not(:last-child),
+table td:not(:last-child) {
+  width: calc(70% / (var(--col-count, 1)));
+  /* --col-count should be set to (number of columns - 1) in your table element via JS or React */
+}
+
+/* Responsive tweaks */
+@media (max-width: 700px) {
+  .table-container {
+    padding: 8px;
+    border-radius: 8px;
+  }
+  table {
+    font-size: 0.95rem;
+  }
+  th, td {
+    padding: 8px 4px;
+  }
+}

SearchFrontend/search_ui/src/components/StatusDisplay.jsx

@@ -1,28 +1,127 @@
-
+import { useTable } from "react-table";
-
+import { useMemo } from "react";
-
+import "./StatusDisplay.css";
-
-
-
 
 export default function StatusesDisplayHUD({ statusMessages }) {
-  
-
     const msg = {};
+    const dataPre = {};
+    const columns = [
+        { Header: "When", accessor: "WHEN" },
+        { Header: "Scheduled", accessor: "SCHEDULED" },
+        { Header: "Queued", accessor: "QUEUED" },
+        { Header: "Processing", accessor: "VECTOR_CALC" },
+        { Header: "Calculating", accessor: "SCORE_CALC" },
+        { Header: "Status", accessor: "STATUS" },
+    ];
 
     statusMessages.forEach((m) => {
-        let when_key = 'other'
+        let when_key = "other";
-        if (m['task'] == 'SCHEDULED')
+        if (m["task"] == "SCHEDULED")
-            m['when'].forEach(( w ) => {   msg[w] = 'Scheduled' })
+            m["when"].forEach((w) => {
-        else {
+                msg[w] = "Scheduled";
-            if ('when' in m)
+                dataPre[w] = {
-                when_key = m['when']
+                    SCHEDULED: "✓",
-            msg[when_key] = m['task']
+                    QUEUED: "",
-        }
+                    VECTOR_CALC: "",
-        
+                    SCORE_CALC: "",
-        
+                    STATUS: "",
-        
+                };
             });
+        else {
+            if ("when" in m) when_key = m["when"];
+            let c_task = m["task"];
+
+            let msg_show;
+            switch (c_task) {
+                case "SCHEDULED":
+                    msg_show = "Scheduled";
+                    dataPre[when_key]["SCHEDULED"] = "✓";
+                    break;
+                case "QUEUEING":
+                    msg_show = "In compute queue";
+                    dataPre[when_key]["QUEUED"] = "✓";
+                    break;
+                case "SCORE_CALC_IN_FOLDER_START":
+                    msg_show = "Calculating Scores";
+                    dataPre[when_key]["VECTOR_CALC"] = "...";
+                    break;
+                case "VECTOR_CALC_IN_FOLDER_START":
+                    msg_show = "Started processing videos";
+                    dataPre[when_key]["VECTOR_CALC"] = "...";
+                    break;
+                case "VECTOR_CALC_IN_FOLDER_BUMP":
+                    msg_show =
+                        "Processing videos: " +
+                        m["progress"] +
+                        "/" +
+                        m["how_many"];
+                    dataPre[when_key]["VECTOR_CALC"] =
+                        m["progress"] + "/" + m["how_many"];
+                    break;
+                case "VECTOR_CALC_IN_FOLDER_DONE":
+                    msg_show = "Finished processing videos";
+                    dataPre[when_key]["VECTOR_CALC"] = "✓";
+                    break;
+
+                case "SCORE_CALC_IN_FOLDER_DONE":
+                    msg_show = "Finished calculating scores";
+                    dataPre[when_key]["VECTOR_CALC"] = "✓";
+                    dataPre[when_key]["SCORE_CALC"] = "✓";
+                    break;
+                default:
+                    msg_show = c_task;
+            }
+            msg[when_key] = msg_show;
+            console.log(when_key);
+            dataPre[when_key]["STATUS"] = msg_show;
+        }
+    });
+
+    Object.entries(dataPre).forEach(([k, v]) => {
+        v["WHEN"] = k;
+    });
+    const data = useMemo(() => Object.values(dataPre), [dataPre]);
+    const columnsMemo = useMemo(() => columns, []);
+
+    const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } =
+        useTable({ columns: columnsMemo, data });
+
+    return (
+        <div className="table-container">
+            <table
+                {...getTableProps()}
+                style={{ "--col-count": columns.length - 1 }}
+            >
+                {rows.length > 0 && (
+                    <thead>
+                        {headerGroups.map((headerGroup) => (
+                            <tr {...headerGroup.getHeaderGroupProps()}>
+                                {headerGroup.headers.map((column) => (
+                                    <th {...column.getHeaderProps()}>
+                                        {column.render("Header")}
+                                    </th>
+                                ))}
+                            </tr>
+                        ))}
+                    </thead>
+                )}
+                <tbody {...getTableBodyProps()}>
+                    {rows.map((row) => {
+                        prepareRow(row);
+                        return (
+                            <tr {...row.getRowProps()}>
+                                {row.cells.map((cell) => (
+                                    <td {...cell.getCellProps()}>
+                                        {cell.render("Cell")}
+                                    </td>
+                                ))}
+                            </tr>
+                        );
+                    })}
+                </tbody>
+            </table>
+        </div>
+    );
 
     return (
         <div>
@@ -33,13 +132,10 @@ export default function StatusesDisplayHUD({ statusMessages }) {
     );
 }
 
-
 export function StatusDisplay({ when, message }) {
-    let msg_show = ''
+    let msg_show = "";
-    
-    msg_show = when + ': ' + message
-
 
+    msg_show = when + ": " + message;
 
     return (
         <div
@@ -56,12 +152,8 @@ export function StatusDisplay({when, message }) {
             {msg_show}
         </div>
     );
-    
 }
 
-
-
-
 //   <div
 //         className="status-messages"
 //         style={{

VectorService/util/embed_scores.py

@@ -39,7 +39,6 @@ def get_matching_file_for_tstamp(target_tstamp, folder_scores):
     web_name = 'media/'+os.path.basename(fname)
     return dict(full_path = fname, path=web_name, timeoffset = offset)
 
-
 def get_vec_rep_file_loc(c_dir):
     vec_rep_file = os.path.join(c_dir, 'vec_rep.npz')
     return vec_rep_file
@@ -92,7 +91,7 @@ def get_vector_representation(c_dir, force_compute = False, redis_key = 'compute
             all_tstamps.append( [x.timestamp() for x in hh['frame_time']])
             enu +=1
 
-        message = {'task':'VECTOR_CALC_IN_FOLDER_BUMP', 'progress': idx+1,  'how_many': len(sorted_videos), 'time': dt.datetime.now().timestamp()}
+        message = {'task':'VECTOR_CALC_IN_FOLDER_BUMP', 'when': c_dir, 'progress': idx+1,  'how_many': len(sorted_videos), 'time': dt.datetime.now().timestamp()}
         r.rpush(redis_key, json.dumps(message))
 
     if len(all_cat) == 0:
@@ -104,6 +103,7 @@ def get_vector_representation(c_dir, force_compute = False, redis_key = 'compute
     
     np.savez(vec_rep_file, embeds = all_embeds, idces= all_idces, timestamps = all_times, source_files = all_source)
     message = {'task':'VECTOR_CALC_IN_FOLDER_DONE', 'when': str(c_dir), 'time': dt.datetime.now().timestamp()}
+    r.rpush(redis_key, json.dumps(message))
     return dict( embeds = all_embeds, idces= all_idces, timestamps = all_times, source_files = all_source)
     
 
@@ -113,7 +113,7 @@ def get_vector_representation(c_dir, force_compute = False, redis_key = 'compute
 def get_scores_embedding_c_dir(c_dir, query_vector, redis_key = 'compute_log'):
     vec_rep = get_vector_representation(c_dir, redis_key=redis_key)
     query_scores = (query_vector @ vec_rep['embeds'].T).squeeze()
-    return query_scores
+    return vec_rep, query_scores
 
 @functools.lru_cache
 def get_query_vector(query):
@@ -226,8 +226,8 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v
             pass
 
 
-    vec_rep = get_vector_representation(c_dir, redis_key = redis_key)
+    # vec_rep = get_vector_representation(c_dir, redis_key = redis_key)
-    query_scores = get_scores_embedding_c_dir(c_dir, tuple(query_vector.tolist()[0]), redis_key = redis_key)
+    vec_rep, query_scores = get_scores_embedding_c_dir(c_dir, tuple(query_vector.tolist()[0]), redis_key = redis_key)
 
     video_json_info = list()
     idces_keep = np.where(query_scores > threshold)[0]

VectorService/vector_service.py

@@ -1,3 +1,4 @@
+
 from typing import Union, Optional, List
 from pydantic import BaseModel
 from fastapi import FastAPI, Request, Depends
@@ -24,9 +25,9 @@ r = redis.Redis(host='localhost', port=6379, db=15)
 
 
 class VideosPostRequest(BaseModel):
-    query: str = "A cat and a human",
+    query: str = "A cat and a human"
-    threshold: float = 0.10,
+    threshold: float = 0.10
-    c_dirs: Optional[List[str]] = None,
+    c_dirs: Optional[List[str]] = None
     task_id: str = 'compute_log'
 
 @app.post("/videos.json")
@@ -61,9 +62,9 @@ async def videos_json(
         ]
     
 
-    print(','.join([str(x) for x in c_dirs]))
+    # print(','.join([str(x) for x in c_dirs]))
-    message = {'task':'SCHEDULED','when':[str(x) for x in c_dirs], 'time':time.time()}
+    # message = {'task':'SCHEDULED','when':[str(x) for x in c_dirs], 'time':time.time()}
-    r.rpush(task_id, json.dumps(message))
+    # r.rpush(task_id, json.dumps(message))?
 
 
     for x in c_dirs: