diff --git a/SearchFrontend/search_ui/src/App.css b/SearchFrontend/search_ui/src/App.css index 6077332..dfa904a 100644 --- a/SearchFrontend/search_ui/src/App.css +++ b/SearchFrontend/search_ui/src/App.css @@ -8,8 +8,8 @@ background: #181a20; } -/* Video.js player */ -.video-js-mod { +/* Video.js player styles */ +.video-player-fullscreen { position: fixed; top: 0; left: 0; @@ -17,98 +17,105 @@ height: 100vh; object-fit: contain; background: #000; + } -.vjs-tech { +.video-tech { object-fit: contain; } -/* Main app layout */ +/* Main app container */ .app-container { - display: flex; - flex-direction: column; height: 100vh; - width: 100vw; - margin: 0; + width: 95vw; + max-width: 95vw; + margin: 0 auto; padding: 0; - gap: 12px; - /* background: #181a20; */ + gap: 0; } - -.flex-group { - display: flex; - flex-direction: column; /* or 'row' if you want horizontal grouping */ - flex: 1 1 0; - min-width: 0; -} -/* Section containers */ -.section-box-horiz { - overflow: visible; - flex-direction: row; - display: flex; - align-items: center; - justify-content: center; -} - -/* Section containers */ -.section-box { - flex: 0 0 5%; - overflow: visible; - /* background: #23272f; */ - /* padding: 0; - box-sizing: border-box; - border-radius: 10px; - margin: 0 16px; - box-shadow: 0 2px 8px rgba(0,0,0,0.10); */ - display: flex; - align-items: center; - justify-content: center; -} - -.timeline-container { - flex: 0 0 24%; - overflow: visible; - background: #20232a; - padding: 0; - box-sizing: border-box; - border-radius: 10px; - margin: 0 16px; - box-shadow: 0 2px 8px rgba(0,0,0,0.10); - display: flex; - align-items: center; - justify-content: center; -} - -.section-box:last-of-type { - flex: 1 1 68%; - overflow: hidden; +/* Controls section */ +.controls-section { + width: 100%; + margin-bottom: 12px; background: #23272f; - padding: 0; - box-sizing: border-box; border-radius: 10px; - margin: 0 16px 16px 16px; - box-shadow: 0 2px 8px rgba(0,0,0,0.10); - display: flex; - align-items: center; - justify-content: center; + padding: 16px 0; + text-align: center; + box-sizing: border-box; } -/* Responsive tweaks */ +/* Control group wrapper */ +.control-group { + display: inline-block; + vertical-align: middle; + min-width: 10%; + margin: 0 8px; +} + +/* Status display section */ +.status-section { + width: 100%; + margin-bottom: 12px; +} + +/* Timeline container */ +.timeline-section { + width: 100%; + height: 30vh; + min-height: 200px; + background: #20232a; + border-radius: 10px; + margin: 0 auto; + box-shadow: 0 2px 8px rgba(0,0,0,0.10); + text-align: center; + box-sizing: border-box; + margin-bottom: 12px; + padding: 0; +} + +/* Video player section */ +.video-section { + width: 100%; + height: 60vw; + margin-top: 12px; + min-height: 40vw; + background: #23272f; + border-radius: 10px; + margin: 0 auto; + box-shadow: 0 2px 8px rgba(0,0,0,0.10); + text-align: center; + box-sizing: border-box; + margin-bottom: 12px; + padding: 0; +} + +/* Date range picker styles */ +.date-picker-wrapper { + max-width: 98vw; + padding: 12px 8px; + border-radius: 8px; +} + +/* Responsive styles */ @media (max-width: 600px) { .app-container { - gap: 6px; + gap: 0; } - .section-box, - .timeline-container, - .section-box:last-of-type { - margin: 0 4px; + + .video-section, + .timeline-section { + margin: 0 4px 8px 4px; border-radius: 6px; - padding: 0; } - .date-range-selector { + + .date-picker-wrapper { max-width: 98vw; padding: 12px 8px; border-radius: 8px; } -} + + .controls-section { + padding: 8px 0; + } +} \ No newline at end of file diff --git a/SearchFrontend/search_ui/src/App.jsx b/SearchFrontend/search_ui/src/App.jsx index 017eab6..2f52262 100644 --- a/SearchFrontend/search_ui/src/App.jsx +++ b/SearchFrontend/search_ui/src/App.jsx @@ -155,6 +155,8 @@ function App() { newData["videos"] = newData["videos"].filter( (vid) => vid["embed_scores"]["score"][1] >= floatValue ); + newData['threshold'] = floatValue; + console.log(newData['threshold']) setDataResults(newData); } @@ -209,22 +211,59 @@ function App() { ); } } - - // Memoize the timeline click handler - const handleTimelineClick = useCallback( - (path, timeoffset) => { - if (playerRef.current && playerInstanceRef.current) { - playerInstanceRef.current.src({ - src: "api/" + path, - type: "video/mp4", - }); - playerInstanceRef.current.on("loadedmetadata", () => { - playerInstanceRef.current.currentTime(timeoffset); - }); - } - }, - [] // Empty dependency array since it only uses playerRef - ); +const handleTimelineClick = useCallback( + (path, timeoffset) => { + if (playerRef.current && playerInstanceRef.current) { + const player = playerInstanceRef.current; + + console.log("Setting video source:", "api/" + path); + console.log("Target time offset:", timeoffset); + + // Clear any existing source first + player.reset(); + + player.src({ + src: "api/" + path, + type: "video/mp4", + }); + + player.load(); + + // Add multiple event listeners for debugging + player.one("loadedmetadata", () => { + console.log("Video metadata loaded"); + console.log("Video duration:", player.duration()); + player.currentTime(timeoffset); + + // Try to play after setting time + const playPromise = player.play(); + if (playPromise !== undefined) { + playPromise.then(() => { + console.log("Video started playing"); + }).catch(error => { + console.error("Error playing video:", error); + }); + } + }); + + player.one("error", (e) => { + console.error("Video error:", e); + console.error("Player error:", player.error()); + }); + + player.one("loadstart", () => { + console.log("Load started"); + }); + + player.one("canplay", () => { + console.log("Video can start playing"); + }); + } else { + console.error("Player ref not available"); + } + }, + [] +); useEffect(() => { const params = new URLSearchParams(window.location.search); @@ -263,12 +302,8 @@ function App() { return (
-
-
+
+
-
-
- setQueryText(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") handleResubmit(); - }} - style={{ - marginLeft: "16px", - marginRight: "16px", - padding: "8px", - borderRadius: "4px", - border: "1px solid #343a40", - color: "#fff", - backgroundColor: "#23272f", - width: "100%", - minWidth: 0, - boxSizing: "border-box", - fontSize: "1.1em", - transition: "width 0.2s", - }} - ref={inputRef} - size={Math.max(queryText.length, 1)} - /> -
+
+ setQueryText(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") handleResubmit(); + }} + style={{ + // padding: "8px", + // borderRadius: "4px", + // border: "1px solid #343a40", + // color: "#fff", + // backgroundColor: "#23272f", + // width: "100%", + // minWidth: 0, + // boxSizing: "border-box", + fontSize: "1.1em", + // transition: "width 0.2s", + }} + ref={inputRef} + size={Math.max(queryText.length, 25)} + />
- +
- - {sliderValue.toFixed(2)} - + {sliderValue.toFixed(2)}
-
+ +
-
+
-
+ +
Loading....
; + // Use useRef instead of props/state to avoid re-renders + const zoomed_range = useRef([null, null]); + const lineDataRef = useRef(null); + const isZoomedInRef = useRef(false); + const hasLineSeriesRef = useRef(false); - // --- Constants --- - const BREAK_GAP = 0; - console.log("REDRAW"); + // --- Early return if loading --- + if (!data_in) return
Loading....
; - const timeFormatOptions = { - withSeconds: { - month: "short", - day: "numeric", - hour: "numeric", - minute: "2-digit", - second: "numeric", - hour12: true, - }, - edges: { - month: "short", - day: "numeric", - hour: "numeric", - minute: "2-digit", - hour12: true, - }, - within: { - hour: "numeric", - minute: "2-digit", - hour12: true, - }, - }; + // --- Constants --- + const BREAK_GAP = 0; + const ZOOM_THRESHOLD = 4 * 60 * 60 * 1000; // 1 hour in ms - const mean = (data) => { - if (data.length < 1) { - return; - } - return data.reduce((prev, current) => prev + current) / data.length; - }; + const fetchLineData = async (startTime, endTime) => { + try { + const params = new URLSearchParams(); + params.append("start_time", new Date(startTime).toISOString()); + params.append("end_time", new Date(endTime).toISOString()); + const response = await fetch( + "/api/line_data.json?" + params.toString() + ); - function prepareVideoData(videos) { - let new_data = []; - videos.forEach((item) => { - let start_time = new Date(1000 * item["start_time"]); - if ("embed_scores" in item) { - var mean_val = item["embed_scores"]["time"] / 2; - // var max_score = Math.max(...item['embed_scores']['score']) - var max_score = item["embed_scores"]["score"][1]; - var max_score_time = new Date( - start_time.getTime() + 1000 * item["embed_scores"]["score"][3] - ); - var new_time = new Date(start_time.getTime() + 1000 * 2 * mean_val); - new_data.push([ - new Date(start_time.getTime()), - new_time, - max_score, - max_score_time, + if (!response.ok) throw new Error(`HTTP error: ${response.status}`); + const data = await response.json(); + + return data["line_data"].map((d) => [d[0] * 1000, d[1]]); + } catch (error) { + console.error("Failed to fetch line data:", error); + return null; + } + }; + + const addLineToChart = (chart, lineData) => { + let vs = chart.getOption().series; + + var ser_mod = vs[2]; + ser_mod.data = fillNulls(lineData).map((d) => [ + virtualTime(new Date(d[0])), + d[1], ]); - // new_data.push([new_time, item['embed_scores']['score'][idx]]); - - // Math.max.apply(Math, item['embed_scores']['time'].map(function(o) { return o.y; })) - // item['embed_scores']['time'].forEach((sec, idx) => { - // let new_time = new Date(start_time.getTime() + 1000 * sec); - - // new_data.push([new_time, item['embed_scores']['score'][idx]]); - // }); - } - }); - - // Remove duplicates and sort - return Array.from(new Set(new_data.map(JSON.stringify)), JSON.parse).sort( - (a, b) => new Date(a[0]) - new Date(b[0]) - ); - } - - function calculateBreaks(videos) { - const breaks = []; - if (videos.length < 3) { - return breaks; - } - let t_diff = videos.at(-1)["end_time"] - videos[0]["start_time"]; - - for (let i = 0; i < videos.length - 1; i++) { - let end_now = videos[i]["end_time"]; - let start_next = videos[i + 1]["start_time"]; - if (start_next - end_now > 60 * 60) { - // still in unix timestamp. break only if spaces of 60 minutes - breaks.push([end_now, start_next]); - } - } - - return breaks; - } - - function fillNulls(data) { - const with_nulls = []; - for (let i = 0; i < data.length; i++) { - with_nulls.push(data[i]); - if (i < data.length - 1) { - const curr_time = new Date(data[i][0]).getTime(); - const next_time = new Date(data[i + 1][0]).getTime(); - if (next_time - curr_time > 1000) { - // with_nulls.push([new Date(curr_time + 1), null]); - // with_nulls.push([new Date(curr_time + 1), 0]); - } - } - } - return with_nulls; - } - - function prepareBreaks(breaksRaw) { - return breaksRaw.map(([start, end]) => ({ - start: new Date(1000 * start), - end: new Date(1000 * end), - gap: BREAK_GAP, - isExpanded: false, - })); - } - function buildVirtualTimeMapper(breaks) { - const sortedBreaks = breaks.slice().sort((a, b) => a.start - b.start); - return function (realDate) { - let offset = 0; - let realMs = realDate.getTime(); - for (const br of sortedBreaks) { - if (realMs >= br.end.getTime()) { - offset += br.end.getTime() - br.start.getTime(); - } else if (realMs > br.start.getTime()) { - offset += realMs - br.start.getTime(); - break; - } - } - return realMs - offset; - }; - } - - function mapVirtualToRealTime(virtualMs, breaks, virtualTime) { - let realMs = virtualMs; - for (const br of breaks) { - const breakStartVirtual = virtualTime(br.start); - const breakDuration = br.end.getTime() - br.start.getTime(); - if (virtualMs >= breakStartVirtual) { - realMs += breakDuration; - } - } - return realMs; - } - - function buildSeries(item, idx) { - const data = item.map(function (item, index) { - return { - value: item, - }; - }); - - console.log(data) - - return { - type: "custom", - renderItem: function (params, api) { - var yValue = api.value(2); - var start = api.coord([api.value(0), yValue]); - var size = api.size([api.value(1) - api.value(0), yValue]); - var style = api.style(); - var maxTime = api.coord([api.value(3), yValue]); - return { - type: "group", - children: [ - { - type: "rect", - shape: { - x: start[0], - y: start[1], - width: size[0], - height: size[1], - }, - style: { fill: "#00F0003F" }, - }, - { - type: "circle", - shape: { cx: maxTime[0], cy: maxTime[1], r: 1 }, - style: { fill: "#00F0003F" }, - }, - ], - }; - }, - symbol: "none", - smooth: true, - large: true, - lineStyle: { normal: { color: "green", width: 1 } }, - // data: item.map(d => [d[0], d[1], d[2], d[3]]), - data: data, - // sampling: 'lttb', - triggerLineEvent: true, - z: 11, - }; - } - - function buildInvisibleHitBoxSeries(item, idx) { - const data = item.map(function (item, index) { - return { - value: item, - }; - }); - - return { - type: "custom", - renderItem: function (params, api) { - var yValue = api.value(2); - var start = api.coord([api.value(0), yValue]); - var size = api.size([api.value(1) - api.value(0), yValue]); - var style = api.style(); - - var maxTime = api.coord([api.value(3), yValue]); - return { - type: "group", - children: [ - { - type: "rect", - shape: { - x: start[0], - y: start[1], - width: size[0], - height: size[1], - }, - style: { fill: "#00F0003F" }, - }, - { - type: "circle", - shape: { cx: maxTime[0], cy: maxTime[1], r: 1 }, - style: { fill: "#00F0003F" }, - }, - ], - }; - }, - symbol: "none", - smooth: true, - // large: true, - lineStyle: { normal: { color: "green", width: 1 } }, - // data: item.map(d => [d[0], d[1], d[2], d[3]]), - data: data, - // sampling: 'lttb', - triggerLineEvent: true, - z: 11, + chart.setOption({ + series: [{}, {}, ser_mod], + }); }; - // return { - // type: 'line', - // symbol: 'none', - // smooth: true, - // large: true, - // lineStyle: { width: 100, opacity: 0 }, - // data: item.map(d => [d[0], d[1]]), - // sampling: 'lttb', - // triggerLineEvent: true, - // z: 10 - // }; - } + const removeLineFromChart = (chart) => { + let vs = chart.getOption().series; - function buildBlankSeries() { - return { - type: "line", - symbol: "none", - lineStyle: { width: 100, opacity: 0 }, - data: [], - z: 4, + var ser_mod = vs[2]; + ser_mod.data = []; + chart.setOption({ + series: [{}, {}, ser_mod], + }); }; - } - // --- Data Processing --- - const videoData = prepareVideoData(data_in["videos"]); - const withNulls = videoData; - data_in["calc_breaks"] = calculateBreaks(data_in["videos"]); + const handleDataZoom = async (params, chart) => { + if (!params || !chart) return; - // const withNulls = fillNulls(videoData); - const breaks = prepareBreaks(data_in["calc_breaks"]); - const virtualTime = buildVirtualTimeMapper(breaks); + console.log(zoomed_range.current); + let plot_start_time = zoomed_range.current[0]; + let plot_end_time = zoomed_range.current[1]; - const breaks_split = data_in["calc_breaks"].flat(1).map(function (x) { - return x * 1000; - }); - // if (videoData.length > 2) { - // breaks_split.unshift(new Date(videoData[0][0]).getTime()) - // breaks_split.push(new Date(videoData.at(-1)[0]).getTime()) - // } - const paired_splits = []; - for (let i = 0; i < breaks_split.length; i += 2) { - paired_splits.push([ - breaks_split[i], - breaks_split[i + 1], - breaks_split[i] / 2 + breaks_split[i + 1] / 2, - ]); - } - const split_centers = paired_splits.map((d) => new Date(d[2])); - const splitCenterVirtualTimes = split_centers.map((d) => virtualTime(d)); - const splitCenterLabels = split_centers.map((d) => - new Date(d).toLocaleTimeString("en-US", timeFormatOptions.edges) - ); + const option = chart.getOption(); + const dataZoom = option.dataZoom?.[0]; + if ( + dataZoom && + dataZoom.startValue !== undefined && + dataZoom.endValue !== undefined + ) { + const zoomRange = dataZoom.endValue - dataZoom.startValue; + const shouldShowLine = zoomRange < ZOOM_THRESHOLD; - const splitCenterMarkLines = splitCenterVirtualTimes.map((vt, i) => ({ - xAxis: vt, - // make the line invisible - lineStyle: { width: 0, color: "transparent" }, - // show the precomputed text - label: { - show: true, - formatter: splitCenterLabels[i], - position: "end", // try other values if overlap; 'end', 'insideStartTop', etc. - color: "#FFFFFF", - fontSize: 11, - rotate: 90, - }, - })); - - const virtualData = withNulls.map( - ([realStartTime, realEndTime, value, realMaxTime]) => [ - virtualTime(new Date(realStartTime)), - virtualTime(new Date(realEndTime)), - value, - virtualTime(new Date(realMaxTime)), - ] - ); - const result = [virtualData]; - const ymax = Math.max(...virtualData.map((d) => d[2])); - // --- Series --- - const seriesNormal = result.map(buildSeries); - // const seriesInvisible = result.map(buildInvisibleHitBoxSeries); - const series_out = [].concat(seriesNormal, buildBlankSeries()); - - // --- Break MarkLines --- - const breakMarkLines = breaks.map((br) => ({ - xAxis: virtualTime(br.start), - lineStyle: { type: "dashed", color: "#888", width: 2 }, - label: { - show: true, - formatter: "Break", - position: "bottom", - color: "#888", - fontSize: 10, - }, - })); - - // Attach break mark lines to the first series - if (seriesNormal[0]) { - seriesNormal[0].markLine = { - symbol: ["none", "none"], - data: [...(breakMarkLines || []), ...(splitCenterMarkLines || [])], - lineStyle: { type: "dashed", color: "#888", width: 2 }, - label: { show: true, position: "bottom", color: "#888", fontSize: 10 }, - }; - } - - // --- Axis & Chart Option --- - const virtual_x_min = virtualData.length > 0 ? virtualData[0][0] : 0; - const virtual_x_max = - virtualData.length > 0 ? virtualData[virtualData.length - 1][0] : 1; - - const option = { - animation: false, - // progressive: 0, // Disable progressive rendering - progressiveThreshold: 100000 , // Disable progressive threshold - mappers: { - virtual_to_real: mapVirtualToRealTime, - real_to_virtual: virtualTime, - }, - response: true, - grid: { - top: 30, // Remove top padding - left: 10, - right: 20, - bottom: 60, - containLabel: true, - }, - dataZoom: [ - { - type: "slider", - show: true, - xAxisIndex: [0], - startValue: virtual_x_min, - endValue: virtual_x_max, - filterMode: 'weakFilter', - }, - { - type: "inside", - xAxisIndex: [0], - startValue: virtual_x_min, - endValue: virtual_x_max, - filterMode: 'weakFilter', - }, - ], - xAxis: { - type: "value", - min: virtual_x_min, - max: virtual_x_max, - splitLine: { show: false }, - axisLabel: { - formatter: function (virtualMs) { - let range = virtual_x_max - virtual_x_min; - if ( - chartRef && - chartRef.current && - chartRef.current.getEchartsInstance - ) { - const chart = chartRef.current.getEchartsInstance(); - const dz = chart.getOption().dataZoom?.[0]; - if ( - dz && - dz.startValue !== undefined && - dz.endValue !== undefined - ) { - range = dz.endValue - dz.startValue; + let offset_start = dataZoom.startValue - plot_start_time; + let offset_end = dataZoom.endValue - plot_end_time; + console.log(offset_start, offset_end); + if (offset_start < 0 || offset_end > 0) { + console.log("Do force getting data"); + isZoomedInRef.current = false; } - } - const realTime = mapVirtualToRealTime(virtualMs, breaks, virtualTime); - if (realTime) { - const useSeconds = range < 5 * 60 * 1000; - const fmt = useSeconds - ? timeFormatOptions.withSeconds - : timeFormatOptions.edges; - return new Date(realTime).toLocaleTimeString("en-US", fmt); - } - return ""; + + if (shouldShowLine && !isZoomedInRef.current) { + // console.log("ZOOMED IN"); + isZoomedInRef.current = true; + let off = 1000 * 60 * 60; + zoomed_range.current = [ + dataZoom.startValue - off, + dataZoom.endValue + off, + ]; + const startTime = mapVirtualToRealTime( + dataZoom.startValue - off, + breaks, + virtualTime + ); + const endTime = mapVirtualToRealTime( + dataZoom.endValue + off, + breaks, + virtualTime + ); + + const fetchedLineData = await fetchLineData(startTime, endTime); + lineDataRef.current = fetchedLineData; + + console.log("Fetched updated data"); + // Add line directly to chart without React re-render + addLineToChart(chart, fetchedLineData); + } else if (!shouldShowLine && isZoomedInRef.current) { + console.log("zoomed out"); + isZoomedInRef.current = false; + lineDataRef.current = null; + zoomed_range.current = [null, null]; + // Remove line directly from chart + removeLineFromChart(chart); + } + } + }; + + const timeFormatOptions = { + withSeconds: { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + second: "numeric", + hour12: true, }, - }, - }, - yAxis: { - type: "value", - min: 0.0, - max: ymax, - splitLine: { show: false }, - }, - series: series_out.map((s) => ({ - ...s, - animation: false, // Disable animation for each series - animationDuration: 0, - })), - }; + edges: { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + hour12: true, + }, + within: { + hour: "numeric", + minute: "2-digit", + hour12: true, + }, + }; - // --- Chart Event Handlers --- - async function onChartClick(params, echarts) { - let pixel; - const nativeEvent = params.event.event; + const mean = (data) => { + if (data.length < 1) { + return; + } + return data.reduce((prev, current) => prev + current) / data.length; + }; - // Support both mouse and touch events - if (nativeEvent.touches && nativeEvent.touches.length > 0) { - // Touch event - const rect = nativeEvent.target.getBoundingClientRect(); - const touch = nativeEvent.touches[0]; - pixel = [ - touch.clientX - rect.left, - touch.clientY - rect.top - ]; - } else { - // Mouse event - pixel = [nativeEvent.offsetX, nativeEvent.offsetY]; + function prepareVideoData(videos) { + let new_data = []; + videos.forEach((item) => { + let start_time = new Date(1000 * item["start_time"]); + if ("embed_scores" in item) { + var mean_val = item["embed_scores"]["time"] / 2; + // var max_score = Math.max(...item['embed_scores']['score']) + var max_score = item["embed_scores"]["score"][1]; + var max_score_time = new Date( + start_time.getTime() + + 1000 * item["embed_scores"]["score"][2] + ); + var new_time = new Date( + start_time.getTime() + 1000 * 2 * mean_val + ); + new_data.push([ + new Date(start_time.getTime()), + new_time, + max_score, + max_score_time, + ]); + // new_data.push([new_time, item['embed_scores']['score'][idx]]); + + // Math.max.apply(Math, item['embed_scores']['time'].map(function(o) { return o.y; })) + // item['embed_scores']['time'].forEach((sec, idx) => { + // let new_time = new Date(start_time.getTime() + 1000 * sec); + + // new_data.push([new_time, item['embed_scores']['score'][idx]]); + // }); + } + }); + + // Remove duplicates and sort + return Array.from( + new Set(new_data.map(JSON.stringify)), + JSON.parse + ).sort((a, b) => new Date(a[0]) - new Date(b[0])); } - const dataCoord = echarts.convertFromPixel({ seriesIndex: 0 }, pixel); + function calculateBreaks(videos) { + const breaks = []; + if (videos.length < 3) { + return breaks; + } + let t_diff = videos.at(-1)["end_time"] - videos[0]["start_time"]; - const res = await fetch("/api/events/click", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - timestamp: mapVirtualToRealTime(dataCoord[0], breaks, virtualTime) / 1000, - }), + for (let i = 0; i < videos.length - 1; i++) { + let end_now = videos[i]["end_time"]; + let start_next = videos[i + 1]["start_time"]; + if (start_next - end_now > 60 * 60) { + // still in unix timestamp. break only if spaces of 60 minutes + breaks.push([end_now, start_next]); + } + } + + return breaks; + } + + function fillNulls(data) { + const with_nulls = []; + for (let i = 0; i < data.length; i++) { + with_nulls.push(data[i]); + if (i < data.length - 1) { + const curr_time = new Date(data[i][0]).getTime(); + const next_time = new Date(data[i + 1][0]).getTime(); + if (next_time - curr_time > 2000) { + with_nulls.push([new Date(curr_time + 1), null]); + // with_nulls.push([new Date(curr_time + 1), 0]); + } + } + } + return with_nulls; + } + + function prepareBreaks(breaksRaw) { + return breaksRaw.map(([start, end]) => ({ + start: new Date(1000 * start), + end: new Date(1000 * end), + gap: BREAK_GAP, + isExpanded: false, + })); + } + function buildVirtualTimeMapper(breaks) { + const sortedBreaks = breaks.slice().sort((a, b) => a.start - b.start); + return function (realDate) { + let offset = 0; + let realMs = realDate.getTime(); + for (const br of sortedBreaks) { + if (realMs >= br.end.getTime()) { + offset += br.end.getTime() - br.start.getTime(); + } else if (realMs > br.start.getTime()) { + offset += realMs - br.start.getTime(); + break; + } + } + return realMs - offset; + }; + } + + function mapVirtualToRealTime(virtualMs, breaks, virtualTime) { + let realMs = virtualMs; + for (const br of breaks) { + const breakStartVirtual = virtualTime(br.start); + const breakDuration = br.end.getTime() - br.start.getTime(); + if (virtualMs >= breakStartVirtual) { + realMs += breakDuration; + } + } + return realMs; + } + + const ymin = Math.floor((data_in.threshold ?? 0.0) * 100) / 100; + function buildSeries(item, idx) { + const data = item.map(function (item, index) { + return { + value: item, + }; + }); + + return { + type: "custom", + renderItem: function (params, api) { + var yValue = api.value(2); + var start = api.coord([api.value(0), yValue]); + var size = api.size([ + api.value(1) - api.value(0), + yValue - ymin, + ]); + var style = api.style(); + var maxTime = api.coord([api.value(3), yValue]); + return { + type: "group", + children: [ + { + type: "rect", + shape: { + x: start[0], + y: start[1], + width: size[0], + height: size[1], + }, + style: { fill: "#00F0003F" }, + }, + { + type: "circle", + shape: { cx: maxTime[0], cy: maxTime[1], r: 1 }, + style: { fill: "#FFFF00" }, + }, + ], + }; + }, + symbol: "none", + smooth: true, + large: true, + lineStyle: { normal: { color: "green", width: 1 } }, + // data: item.map(d => [d[0], d[1], d[2], d[3]]), + data: data, + // sampling: 'lttb', + triggerLineEvent: true, + z: 11, + }; + } + + function buildBlankSeries() { + return { + type: "line", + symbol: "none", + lineStyle: { width: 100, opacity: 0 }, + data: [], + smooth: true, + sampling: 'lttb', + z: 4, + }; + } + + // --- Data Processing --- + const videoData = prepareVideoData(data_in["videos"]); + const withNulls = videoData; + data_in["calc_breaks"] = calculateBreaks(data_in["videos"]); + + // const withNulls = fillNulls(videoData); + const breaks = prepareBreaks(data_in["calc_breaks"]); + const virtualTime = buildVirtualTimeMapper(breaks); + + const breaks_split = data_in["calc_breaks"].flat(1).map(function (x) { + return x * 1000; }); - if (!res.ok) throw new Error(`HTTP error: ${res.status}`); - const { path, timeoffset } = await res.json(); - if (onTimelineClick) - onTimelineClick(path, virtualTime(new Date(timeoffset))); - } + // if (videoData.length > 2) { + // breaks_split.unshift(new Date(videoData[0][0]).getTime()) + // breaks_split.push(new Date(videoData.at(-1)[0]).getTime()) + // } + const paired_splits = []; + for (let i = 0; i < breaks_split.length; i += 2) { + paired_splits.push([ + breaks_split[i], + breaks_split[i + 1], + breaks_split[i] / 2 + breaks_split[i + 1] / 2, + ]); + } + const split_centers = paired_splits.map((d) => new Date(d[2])); + const splitCenterVirtualTimes = split_centers.map((d) => virtualTime(d)); + const splitCenterLabels = split_centers.map((d) => + new Date(d).toLocaleTimeString("en-US", timeFormatOptions.edges) + ); - function onChartReady(echarts) { - // Chart is ready - } + const splitCenterMarkLines = splitCenterVirtualTimes.map((vt, i) => ({ + xAxis: vt, + // make the line invisible + lineStyle: { width: 0, color: "transparent" }, + // show the precomputed text + label: { + show: true, + formatter: splitCenterLabels[i], + position: "end", // try other values if overlap; 'end', 'insideStartTop', etc. + color: "#FFFFFF", + fontSize: 11, + rotate: 90, + }, + })); - const onEvents = { click: onChartClick }; - window.chartRef2 = chartRef; - // --- Render --- - return ( - - ); + const virtualData = withNulls.map( + ([realStartTime, realEndTime, value, realMaxTime]) => [ + virtualTime(new Date(realStartTime)), + virtualTime(new Date(realEndTime)), + value, + virtualTime(new Date(realMaxTime)), + ] + ); + const result = [virtualData]; + + const ymax = Math.max(...virtualData.map((d) => d[2])); + // --- Series --- + const seriesNormal = result.map(buildSeries); + // const seriesInvisible = result.map(buildInvisibleHitBoxSeries); + + const lineSeries = { + id: "zoomLine", + type: "line", + data: [], + lineStyle: { color: "#00FFFF", width: 1 }, + symbol: "none", + z: 20, + animation: false, + triggerLineEvent: true, + xAxisIndex: 0, + connectNulls: false, + yAxisIndex: 0, + }; + + const series_out = [].concat(seriesNormal, buildBlankSeries(), lineSeries); + + // --- Break MarkLines --- + const breakMarkLines = breaks.map((br) => ({ + xAxis: virtualTime(br.start), + lineStyle: { type: "dashed", color: "#888", width: 2 }, + label: { + show: true, + formatter: "Break", + position: "bottom", + color: "#888", + fontSize: 10, + }, + })); + + // Attach break mark lines to the first series + if (seriesNormal[0]) { + seriesNormal[0].markLine = { + symbol: ["none", "none"], + data: [...(breakMarkLines || []), ...(splitCenterMarkLines || [])], + lineStyle: { type: "dashed", color: "#888", width: 2 }, + label: { + show: true, + position: "bottom", + color: "#888", + fontSize: 10, + }, + }; + } + + // --- Axis & Chart Option --- + const virtual_x_min = + (virtualData.length > 0 ? virtualData[0][0] : 0) - 100000; + const virtual_x_max = + (virtualData.length > 0 ? virtualData[virtualData.length - 1][0] : 1) + + 4100000; + + const option = { + animation: false, + // progressive: 0, // Disable progressive rendering + progressiveThreshold: 100000, // Disable progressive threshold + mappers: { + virtual_to_real: mapVirtualToRealTime, + real_to_virtual: virtualTime, + }, + response: true, + grid: { + top: 30, // Remove top padding + left: 10, + right: 20, + bottom: 60, + containLabel: true, + }, + dataZoom: [ + { + type: "slider", + show: true, + xAxisIndex: [0], + startValue: virtual_x_min, + endValue: virtual_x_max, + filterMode: "weakFilter", + }, + { + type: "inside", + xAxisIndex: [0], + startValue: virtual_x_min, + endValue: virtual_x_max, + filterMode: "weakFilter", + }, + ], + xAxis: { + type: "value", + min: virtual_x_min, + max: virtual_x_max, + splitLine: { show: false }, + axisLabel: { + formatter: function (virtualMs) { + let range = virtual_x_max - virtual_x_min; + if ( + chartRef && + chartRef.current && + chartRef.current.getEchartsInstance + ) { + const chart = chartRef.current.getEchartsInstance(); + const dz = chart.getOption().dataZoom?.[0]; + if ( + dz && + dz.startValue !== undefined && + dz.endValue !== undefined + ) { + range = dz.endValue - dz.startValue; + } + } + const realTime = mapVirtualToRealTime( + virtualMs, + breaks, + virtualTime + ); + if (realTime) { + const useSeconds = range < 5 * 60 * 1000; + const fmt = useSeconds + ? timeFormatOptions.withSeconds + : timeFormatOptions.edges; + return new Date(realTime).toLocaleTimeString( + "en-US", + fmt + ); + } + return ""; + }, + }, + }, + yAxis: { + type: "value", + min: Math.floor(ymin * 100) / 100, + max: Math.ceil(ymax * 100) / 100, + splitLine: { show: false }, + }, + series: series_out.map((s) => ({ + ...s, + animation: false, // Disable animation for each series + animationDuration: 0, + xAxisIndex: 0, // Explicitly set axis index for all series + yAxisIndex: 0, + })), + }; + + // --- Chart Event Handlers --- + async function onChartClick(params, echarts) { + let pixel; + const nativeEvent = params.event.event; + + // Support both mouse and touch events + if (nativeEvent.touches && nativeEvent.touches.length > 0) { + // Touch event + const rect = nativeEvent.target.getBoundingClientRect(); + const touch = nativeEvent.touches[0]; + pixel = [touch.clientX - rect.left, touch.clientY - rect.top]; + } else { + // Mouse event + pixel = [nativeEvent.offsetX, nativeEvent.offsetY]; + } + + const dataCoord = echarts.convertFromPixel({ seriesIndex: 0 }, pixel); + + const res = await fetch("/api/events/click", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + timestamp: + mapVirtualToRealTime(dataCoord[0], breaks, virtualTime) / + 1000, + }), + }); + if (!res.ok) throw new Error(`HTTP error: ${res.status}`); + const { path, timeoffset } = await res.json(); + if (onTimelineClick) + onTimelineClick(path, virtualTime(new Date(timeoffset))); + } + + function onChartReady(echarts) { + // Chart is ready + } + + const onEvents = { click: onChartClick, dataZoom: handleDataZoom }; + window.chartRef2 = chartRef; + // --- Render --- + return ( + + ); }); export default EmbedTimeline; diff --git a/SearchScratch/test_filter.py b/SearchScratch/test_filter.py new file mode 100644 index 0000000..3a120f9 --- /dev/null +++ b/SearchScratch/test_filter.py @@ -0,0 +1,61 @@ +import pickle + + +cache_files = ['/mnt/hdd_24tb_1/videos/ftp/leopards1/2025/09/12/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/mnt/hdd_24tb_1/videos/ftp/leopards1/2025/09/13/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/14/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/15/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/16/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/17/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/18/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl', '/srv/ftp_tcc/leopards1/2025/09/19/embedding_scores@0.0@926895f71538e3683e9af0956af94cf4.pkl'] +import time +from datetime import timedelta, datetime +start_time = time.time() +all_c = list() + +start_time = 1757892175.042 +end_time = 1757894197.548 + +def check_if_overlap(start_1, end_1, start_2, end_2): + ff = sorted([[start_1, end_1],[start_2, end_2]],key=lambda x: x[0]) + return ff[0][1] > ff[1][0] + +def get_cache_data(start_time, end_time, cache_files): + targvals = [start_time, end_time] + for f in cache_files: + fold_start_time = datetime(*[int(x) for x in f.split('/')[-4:-1]]).timestamp() + fold_end_time = fold_start_time + 86400.0 + + has_overlap = check_if_overlap( start_time, end_time, fold_start_time, fold_end_time) + + if not has_overlap: + continue + + print(f'Loading {f}') + + with open(f,'rb') as ff: + all_c.append(pickle.load(ff)) + return all_c + +st = time.time() +all_cach = get_cache_data(start_time, end_time, cache_files) + +vids = list() +for c_c in all_cach: + vids.extend(c_c['videos']) + +data_filt = list() +for v in vids: + if check_if_overlap( v['start_time'], v['end_time'], start_time, end_time): + data_filt.append(v) + + + +time_vec = np.hstack([ np.asarray(f['embed_scores']['time'])+f['start_time'] for f in data_filt]) +score_vec = np.hstack([f['embed_scores']['score'] for f in data_filt]) + + +s_time, s_ind = np.unique(time_vec, return_index=True) +s_score = score_vec[s_ind] + + +out_array = np.asarray([s_time, s_score]).T.tolist() + + + + + diff --git a/VectorService/util/embed_scores.py b/VectorService/util/embed_scores.py index df7ecb1..cc7964c 100644 --- a/VectorService/util/embed_scores.py +++ b/VectorService/util/embed_scores.py @@ -134,20 +134,22 @@ def calculate_embedding_score_in_folders(c_dirs, threshold, query = None, query_ # kwargs = [{'c_dir':x, 'threshold':threshold, 'query': query} for x in c_dirs] args = [(x, threshold, query, None, logger, redis_key) for x in c_dirs] - logger.info(f"CALCULATING FOR {args}") + # logger.info(f"CALCULATING FOR {args}") with Pool(processes=8) as pool: out = pool.starmap(calculate_embedding_score_in_folder, args) - logger.info(f"DONE CALCULATING FOR {args}") + # logger.info(f"DONE CALCULATING FOR {args}") - for x in out: + cache_files = list(); + for x, cache_file_loc in out: try: result_list.extend(x['videos']) + cache_files.append(cache_file_loc); except Exception as e: print(e, x) - return {'videos':result_list} + return {'videos':result_list, 'cache_file_locs': cache_files} def collapse_scores_to_maxmin_avg(folder_scores): @@ -215,7 +217,7 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v logger.info(f"LOADED EMBEDDING SCORE FROM CACHE {cache_file_loc}") message = {'task':'SCORE_CALC_IN_FOLDER_DONE', 'when': str(c_dir), 'time': dt.datetime.now().timestamp(), 'precomputed': True} r.rpush(redis_key, json.dumps(message)) - return video_json_info + return (video_json_info, cache_file_loc) else: logger.info(f"CACHE FILE IS OLD, DELETING VEC REP FILE AND RECREATING {cache_file_loc}") os.remove( get_vec_rep_file_loc(c_dir)) @@ -288,8 +290,8 @@ def calculate_embedding_score_in_folder(og_dir, threshold, query = None, query_v with open(cache_file_loc, 'wb') as f: logger.info(f"WRITING EMBEDDING SCORE TO CACHE {cache_file_loc}") pickle.dump(to_write, f) - - return to_write + logger.info(f"SAVED EMBEDDING SCORE TO CACHE {cache_file_loc}") + return (to_write, cache_file_loc) def get_matching_file_given_filename(web_name, folder_scores): diff --git a/VectorService/vector_service.py b/VectorService/vector_service.py index c721f50..d3bbd2e 100644 --- a/VectorService/vector_service.py +++ b/VectorService/vector_service.py @@ -82,9 +82,6 @@ async def videos_json( folder_scores["breaks"] = ES.add_breaks_between_videos(folder_scores) folder_scores['videos'] = ES.collapse_scores_to_maxmin_avg(folder_scores) - - - session["folder_scores"] = folder_scores return folder_scores diff --git a/scripts/start_backend.sh b/scripts/start_backend.sh new file mode 100755 index 0000000..911b47d --- /dev/null +++ b/scripts/start_backend.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd /home/thebears/Web/Nuggets/SearchInterface/SearchBackend/ + +/home/thebears/envs/embedding_search_web_server/bin/fastapi dev --host 0.0.0.0 --port 5003 /home/thebears/Web/Nuggets/SearchInterface/SearchBackend/backend.py + diff --git a/scripts/start_frontend.sh b/scripts/start_frontend.sh new file mode 100755 index 0000000..a066fe8 --- /dev/null +++ b/scripts/start_frontend.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /home/thebears/Web/Nuggets/SearchInterface/SearchFrontend/search_ui + +npm run dev + + diff --git a/scripts/start_vector.sh b/scripts/start_vector.sh new file mode 100755 index 0000000..55e210b --- /dev/null +++ b/scripts/start_vector.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd /home/thebears/Web/Nuggets/SearchInterface/VectorService +/home/thebears/envs/embedding_search_web_server/bin/fastapi dev --host 0.0.0.0 --port 5004 /home/thebears/Web/Nuggets/SearchInterface/VectorService/vector_service.py + +