From 20a9b210273b276b343fb1a5e77edefe734c5952 Mon Sep 17 00:00:00 2001 From: Yossi Elkrief Date: Mon, 2 Mar 2026 12:57:43 +0200 Subject: [PATCH] feat: Live screen with 3D Gaussian splat WebView --- .../src/assets/webview/gaussian-splats.html | 585 ++++++++++++++++++ .../LiveScreen/GaussianSplatWebView.tsx | 41 ++ mobile/src/screens/LiveScreen/LiveHUD.tsx | 164 +++++ mobile/src/screens/LiveScreen/index.tsx | 215 +++++++ .../screens/LiveScreen/useGaussianBridge.ts | 97 +++ mobile/src/types/html.d.ts | 4 + 6 files changed, 1106 insertions(+) create mode 100644 mobile/src/types/html.d.ts diff --git a/mobile/src/assets/webview/gaussian-splats.html b/mobile/src/assets/webview/gaussian-splats.html index e69de29..16a398e 100644 --- a/mobile/src/assets/webview/gaussian-splats.html +++ b/mobile/src/assets/webview/gaussian-splats.html @@ -0,0 +1,585 @@ + + + + + + + WiFi DensePose Splat Viewer + + + +
+ + + + + + + diff --git a/mobile/src/screens/LiveScreen/GaussianSplatWebView.tsx b/mobile/src/screens/LiveScreen/GaussianSplatWebView.tsx index e69de29..e58568c 100644 --- a/mobile/src/screens/LiveScreen/GaussianSplatWebView.tsx +++ b/mobile/src/screens/LiveScreen/GaussianSplatWebView.tsx @@ -0,0 +1,41 @@ +import { LayoutChangeEvent, StyleSheet } from 'react-native'; +import type { RefObject } from 'react'; +import { WebView, type WebViewMessageEvent } from 'react-native-webview'; +import GAUSSIAN_SPLATS_HTML from '@/assets/webview/gaussian-splats.html'; + +type GaussianSplatWebViewProps = { + onMessage: (event: WebViewMessageEvent) => void; + onError: () => void; + webViewRef: RefObject; + onLayout?: (event: LayoutChangeEvent) => void; +}; + +export const GaussianSplatWebView = ({ + onMessage, + onError, + webViewRef, + onLayout, +}: GaussianSplatWebViewProps) => { + const html = typeof GAUSSIAN_SPLATS_HTML === 'string' ? GAUSSIAN_SPLATS_HTML : ''; + + return ( + + ); +}; + +const styles = StyleSheet.create({ + webView: { + flex: 1, + backgroundColor: '#0A0E1A', + }, +}); diff --git a/mobile/src/screens/LiveScreen/LiveHUD.tsx b/mobile/src/screens/LiveScreen/LiveHUD.tsx index e69de29..19c93ac 100644 --- a/mobile/src/screens/LiveScreen/LiveHUD.tsx +++ b/mobile/src/screens/LiveScreen/LiveHUD.tsx @@ -0,0 +1,164 @@ +import { Pressable, StyleSheet, View } from 'react-native'; +import { memo, useCallback, useState } from 'react'; +import Animated, { useAnimatedStyle, useSharedValue, withTiming } from 'react-native-reanimated'; +import { StatusDot } from '@/components/StatusDot'; +import { ModeBadge } from '@/components/ModeBadge'; +import { ThemedText } from '@/components/ThemedText'; +import { formatConfidence, formatRssi } from '@/utils/formatters'; +import { colors, spacing } from '@/theme'; +import type { ConnectionStatus } from '@/types/sensing'; + +type LiveMode = 'LIVE' | 'SIM' | 'RSSI'; + +type LiveHUDProps = { + rssi?: number; + connectionStatus: ConnectionStatus; + fps: number; + confidence: number; + personCount: number; + mode: LiveMode; +}; + +const statusTextMap: Record = { + connected: 'Connected', + simulated: 'Simulated', + connecting: 'Connecting', + disconnected: 'Disconnected', +}; + +const statusDotStatusMap: Record = { + connected: 'connected', + simulated: 'simulated', + connecting: 'connecting', + disconnected: 'disconnected', +}; + +export const LiveHUD = memo( + ({ rssi, connectionStatus, fps, confidence, personCount, mode }: LiveHUDProps) => { + const [panelVisible, setPanelVisible] = useState(true); + const panelAlpha = useSharedValue(1); + + const togglePanel = useCallback(() => { + const next = !panelVisible; + setPanelVisible(next); + panelAlpha.value = withTiming(next ? 1 : 0, { duration: 220 }); + }, [panelAlpha, panelVisible]); + + const animatedPanelStyle = useAnimatedStyle(() => ({ + opacity: panelAlpha.value, + })); + + const statusText = statusTextMap[connectionStatus]; + + return ( + + + {/* App title */} + + + WiFi-DensePose + + + + {/* Status + FPS */} + + + + + {statusText} + + + {fps > 0 && ( + + {fps} FPS + + )} + + + {/* Bottom panel */} + + + RSSI + + {formatRssi(rssi)} + + + + + + + + + Confidence + + {formatConfidence(confidence)} + + People: {personCount} + + + + + ); + }, +); + +const styles = StyleSheet.create({ + topLeft: { + position: 'absolute', + top: spacing.md, + left: spacing.md, + }, + appTitle: { + color: colors.textPrimary, + }, + topRight: { + position: 'absolute', + top: spacing.md, + right: spacing.md, + alignItems: 'flex-end', + gap: 4, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + gap: spacing.sm, + }, + statusText: { + color: colors.textPrimary, + }, + bottomPanel: { + position: 'absolute', + left: spacing.sm, + right: spacing.sm, + bottom: spacing.sm, + minHeight: 72, + borderRadius: 12, + backgroundColor: 'rgba(10,14,26,0.72)', + borderWidth: 1, + borderColor: 'rgba(50,184,198,0.35)', + paddingHorizontal: spacing.md, + paddingVertical: spacing.sm, + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + }, + bottomCell: { + flex: 1, + alignItems: 'center', + }, + bottomCellRight: { + flex: 1, + alignItems: 'flex-end', + }, + bigValue: { + color: colors.accent, + marginTop: 2, + marginBottom: 2, + }, + metaText: { + color: colors.textPrimary, + marginBottom: 4, + }, +}); + +LiveHUD.displayName = 'LiveHUD'; diff --git a/mobile/src/screens/LiveScreen/index.tsx b/mobile/src/screens/LiveScreen/index.tsx index e69de29..6589863 100644 --- a/mobile/src/screens/LiveScreen/index.tsx +++ b/mobile/src/screens/LiveScreen/index.tsx @@ -0,0 +1,215 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { Button, LayoutChangeEvent, StyleSheet, View } from 'react-native'; +import type { WebView } from 'react-native-webview'; +import type { WebViewMessageEvent } from 'react-native-webview'; +import { ErrorBoundary } from '@/components/ErrorBoundary'; +import { LoadingSpinner } from '@/components/LoadingSpinner'; +import { ThemedText } from '@/components/ThemedText'; +import { ThemedView } from '@/components/ThemedView'; +import { usePoseStream } from '@/hooks/usePoseStream'; +import { colors, spacing } from '@/theme'; +import type { ConnectionStatus, SensingFrame } from '@/types/sensing'; +import { useGaussianBridge } from './useGaussianBridge'; +import { GaussianSplatWebView } from './GaussianSplatWebView'; +import { LiveHUD } from './LiveHUD'; + +type LiveMode = 'LIVE' | 'SIM' | 'RSSI'; + +const getMode = ( + status: ConnectionStatus, + isSimulated: boolean, + frame: SensingFrame | null, +): LiveMode => { + if (isSimulated || frame?.source === 'simulated') { + return 'SIM'; + } + + if (status === 'connected') { + return 'LIVE'; + } + + return 'RSSI'; +}; + +const dispatchWebViewMessage = (webViewRef: { current: WebView | null }, message: unknown) => { + const webView = webViewRef.current; + if (!webView) { + return; + } + + const payload = JSON.stringify(message); + webView.injectJavaScript( + `window.dispatchEvent(new MessageEvent('message', { data: ${JSON.stringify(payload)} })); true;`, + ); +}; + +export const LiveScreen = () => { + const webViewRef = useRef(null); + const { lastFrame, connectionStatus, isSimulated } = usePoseStream(); + const bridge = useGaussianBridge(webViewRef); + + const [webError, setWebError] = useState(null); + const [viewerKey, setViewerKey] = useState(0); + const sendTimeoutRef = useRef | null>(null); + const pendingFrameRef = useRef(null); + const lastSentAtRef = useRef(0); + + const clearSendTimeout = useCallback(() => { + if (!sendTimeoutRef.current) { + return; + } + clearTimeout(sendTimeoutRef.current); + sendTimeoutRef.current = null; + }, []); + + useEffect(() => { + if (!lastFrame) { + return; + } + + pendingFrameRef.current = lastFrame; + const now = Date.now(); + + const flush = () => { + if (!bridge.isReady || !pendingFrameRef.current) { + return; + } + + bridge.sendFrame(pendingFrameRef.current); + lastSentAtRef.current = Date.now(); + pendingFrameRef.current = null; + }; + + const waitMs = Math.max(0, 500 - (now - lastSentAtRef.current)); + + if (waitMs <= 0) { + flush(); + return; + } + + clearSendTimeout(); + sendTimeoutRef.current = setTimeout(() => { + sendTimeoutRef.current = null; + flush(); + }, waitMs); + + return () => { + clearSendTimeout(); + }; + }, [bridge.isReady, lastFrame, bridge.sendFrame, clearSendTimeout]); + + useEffect(() => { + return () => { + dispatchWebViewMessage(webViewRef, { type: 'DISPOSE' }); + clearSendTimeout(); + pendingFrameRef.current = null; + }; + }, [clearSendTimeout]); + + const onMessage = useCallback( + (event: WebViewMessageEvent) => { + bridge.onMessage(event); + }, + [bridge], + ); + + const onLayout = useCallback((event: LayoutChangeEvent) => { + const { width, height } = event.nativeEvent.layout; + if (width <= 0 || height <= 0 || Number.isNaN(width) || Number.isNaN(height)) { + return; + } + + dispatchWebViewMessage(webViewRef, { + type: 'RESIZE', + payload: { + width: Math.max(1, Math.floor(width)), + height: Math.max(1, Math.floor(height)), + }, + }); + }, []); + + const handleWebError = useCallback(() => { + setWebError('Live renderer failed to initialize'); + }, []); + + const handleRetry = useCallback(() => { + setWebError(null); + bridge.reset(); + setViewerKey((value) => value + 1); + }, [bridge]); + + const rssi = lastFrame?.features?.mean_rssi; + const personCount = lastFrame?.classification?.presence ? 1 : 0; + const mode = getMode(connectionStatus, isSimulated, lastFrame); + + if (webError || bridge.error) { + return ( + + Live visualization failed + + {webError ?? bridge.error} + +