feat: handle images instead webcam

.gitignore

@@ -0,0 +1 @@
+node_modules/

src/App.tsx

@@ -1,109 +1,48 @@
-import { useState, useEffect, useRef, useCallback, useMemo } from "react";
+import { useState, useCallback } from "react";
 import LoadingScreen from "./components/LoadingScreen";
-import CaptioningView from "./components/CaptioningView";
-import WelcomeScreen from "./components/WelcomeScreen";
-import WebcamPermissionDialog from "./components/WebcamPermissionDialog";
+import ImageUpload from "./components/ImageUpload";
+import ImageAnalysisView from "./components/ImageAnalysisView";
 import type { AppState } from "./types";
 
 export default function App() {
-  const [appState, setAppState] = useState<AppState>("requesting-permission");
-  const [webcamStream, setWebcamStream] = useState<MediaStream | null>(null);
-  const [isVideoReady, setIsVideoReady] = useState(false);
-  const videoRef = useRef<HTMLVideoElement | null>(null);
+  const [appState, setAppState] = useState<AppState>("upload");
+  const [uploadedImages, setUploadedImages] = useState<File[]>([]);
 
-  const handlePermissionGranted = useCallback((stream: MediaStream) => {
-    setWebcamStream(stream);
-    setAppState("welcome");
-  }, []);
-
-  const handleStart = useCallback(() => {
+  const handleImagesUploaded = useCallback((files: File[]) => {
+    setUploadedImages(files);
     setAppState("loading");
   }, []);
 
   const handleLoadingComplete = useCallback(() => {
-    setAppState("captioning");
+    setAppState("analyzing");
   }, []);
 
-  const playVideo = useCallback(async (video: HTMLVideoElement) => {
-    try {
-      await video.play();
-    } catch (error) {
-      console.error("Failed to play video:", error);
-    }
+  const handleBackToUpload = useCallback(() => {
+    setUploadedImages([]);
+    setAppState("upload");
   }, []);
 
-  const setupVideo = useCallback(
-    (video: HTMLVideoElement, stream: MediaStream) => {
-      video.srcObject = stream;
-
-      const handleCanPlay = () => {
-        setIsVideoReady(true);
-        playVideo(video);
-      };
-
-      video.addEventListener("canplay", handleCanPlay, { once: true });
-
-      return () => {
-        video.removeEventListener("canplay", handleCanPlay);
-      };
-    },
-    [playVideo],
-  );
-
-  useEffect(() => {
-    if (webcamStream && videoRef.current) {
-      const video = videoRef.current;
-
-      video.srcObject = null;
-      video.load();
-
-      const cleanup = setupVideo(video, webcamStream);
-      return cleanup;
-    }
-  }, [webcamStream, setupVideo]);
-
-  const videoBlurState = useMemo(() => {
-    switch (appState) {
-      case "requesting-permission":
-        return "blur(20px) brightness(0.2) saturate(0.5)";
-      case "welcome":
-        return "blur(12px) brightness(0.3) saturate(0.7)";
-      case "loading":
-        return "blur(8px) brightness(0.4) saturate(0.8)";
-      case "captioning":
-        return "none";
-      default:
-        return "blur(20px) brightness(0.2) saturate(0.5)";
-    }
-  }, [appState]);
-
   return (
     <div className="App relative h-screen overflow-hidden">
-      <div className="absolute inset-0 bg-gray-900" />
+      <div className="absolute inset-0 bg-gradient-to-br from-gray-900 via-blue-900/20 to-purple-900/20" />
 
-      {webcamStream && (
-        <video
-          ref={videoRef}
-          autoPlay
-          muted
-          playsInline
-          className="absolute inset-0 w-full h-full object-cover transition-all duration-1000 ease-out"
-          style={{
-            filter: videoBlurState,
-            opacity: isVideoReady ? 1 : 0,
-          }}
+      {appState !== "analyzing" && <div className="absolute inset-0 bg-gray-900/80 backdrop-blur-sm" />}
+
+      {appState === "upload" && (
+        <ImageUpload
+          onImagesUploaded={handleImagesUploaded}
+          isAnalyzing={false}
         />
       )}
 
-      {appState !== "captioning" && <div className="absolute inset-0 bg-gray-900/80 backdrop-blur-sm" />}
-
-      {appState === "requesting-permission" && <WebcamPermissionDialog onPermissionGranted={handlePermissionGranted} />}
-
-      {appState === "welcome" && <WelcomeScreen onStart={handleStart} />}
-
       {appState === "loading" && <LoadingScreen onComplete={handleLoadingComplete} />}
 
-      {appState === "captioning" && <CaptioningView videoRef={videoRef} />}
+      {appState === "analyzing" && (
+        <ImageAnalysisView
+          images={uploadedImages}
+          onBackToUpload={handleBackToUpload}
+        />
+      )}
     </div>
   );
 }

src/components/ImageAnalysisView.tsx

@@ -0,0 +1,207 @@
+import { useState, useRef, useEffect, useCallback } from "react";
+import DraggableContainer from "./DraggableContainer";
+import PromptInput from "./PromptInput";
+import GlassButton from "./GlassButton";
+import GlassContainer from "./GlassContainer";
+import { useVLMContext } from "../context/useVLMContext";
+import { PROMPTS, GLASS_EFFECTS } from "../constants";
+import type { ImageAnalysisResult } from "../types";
+
+interface ImageAnalysisViewProps {
+  images: File[];
+  onBackToUpload: () => void;
+}
+
+export default function ImageAnalysisView({ images, onBackToUpload }: ImageAnalysisViewProps) {
+  const [results, setResults] = useState<ImageAnalysisResult[]>([]);
+  const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default);
+  const [isAnalyzing, setIsAnalyzing] = useState<boolean>(false);
+  const [currentImageIndex, setCurrentImageIndex] = useState<number>(0);
+  const [selectedImageUrl, setSelectedImageUrl] = useState<string>("");
+
+  const { isLoaded, runInference } = useVLMContext();
+  const abortControllerRef = useRef<AbortController | null>(null);
+
+  // Create preview URL for selected image
+  useEffect(() => {
+    if (images[currentImageIndex]) {
+      const url = URL.createObjectURL(images[currentImageIndex]);
+      setSelectedImageUrl(url);
+      return () => URL.revokeObjectURL(url);
+    }
+  }, [images, currentImageIndex]);
+
+  const analyzeAllImages = useCallback(async () => {
+    if (!isLoaded || isAnalyzing) return;
+
+    setIsAnalyzing(true);
+    setResults([]);
+    
+    abortControllerRef.current?.abort();
+    abortControllerRef.current = new AbortController();
+    
+    const analysisResults: ImageAnalysisResult[] = [];
+
+    try {
+      for (let i = 0; i < images.length; i++) {
+        if (abortControllerRef.current.signal.aborted) break;
+        
+        setCurrentImageIndex(i);
+        const file = images[i];
+        
+        try {
+          const caption = await runInference(file, currentPrompt);
+          analysisResults.push({ file, caption });
+        } catch (error) {
+          const errorMsg = error instanceof Error ? error.message : String(error);
+          analysisResults.push({ file, caption: "", error: errorMsg });
+        }
+        
+        setResults([...analysisResults]);
+      }
+    } catch (error) {
+      console.error("Analysis interrupted:", error);
+    } finally {
+      setIsAnalyzing(false);
+    }
+  }, [images, currentPrompt, isLoaded, runInference, isAnalyzing]);
+
+  const handlePromptChange = useCallback((prompt: string) => {
+    setCurrentPrompt(prompt);
+  }, []);
+
+  const handleImageSelect = useCallback((index: number) => {
+    setCurrentImageIndex(index);
+  }, []);
+
+  const stopAnalysis = useCallback(() => {
+    abortControllerRef.current?.abort();
+    setIsAnalyzing(false);
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      abortControllerRef.current?.abort();
+    };
+  }, []);
+
+  return (
+    <div className="absolute inset-0 text-white">
+      {/* Main image display */}
+      <div className="relative w-full h-full flex">
+        {/* Image preview */}
+        <div className="flex-1 flex items-center justify-center p-8">
+          {selectedImageUrl && (
+            <img
+              src={selectedImageUrl}
+              alt={`Preview of ${images[currentImageIndex]?.name}`}
+              className="max-w-full max-h-full object-contain rounded-lg shadow-2xl"
+            />
+          )}
+        </div>
+
+        {/* Sidebar with image thumbnails and results */}
+        <div className="w-80 bg-black/20 backdrop-blur-sm border-l border-white/20 overflow-y-auto">
+          {/* Controls */}
+          <div className="p-4 border-b border-white/20">
+            <div className="flex gap-2 mb-4">
+              <GlassButton onClick={onBackToUpload} className="flex-1">
+                Back to Upload
+              </GlassButton>
+              {!isAnalyzing ? (
+                <GlassButton 
+                  onClick={analyzeAllImages} 
+                  disabled={!isLoaded}
+                  className="flex-1"
+                >
+                  Analyze All
+                </GlassButton>
+              ) : (
+                <GlassButton onClick={stopAnalysis} className="flex-1 bg-red-500/20">
+                  Stop
+                </GlassButton>
+              )}
+            </div>
+            
+            {isAnalyzing && (
+              <div className="text-sm text-white/70 text-center">
+                Analyzing image {currentImageIndex + 1} of {images.length}...
+              </div>
+            )}
+          </div>
+
+          {/* Image list with results */}
+          <div className="p-4 space-y-4">
+            {images.map((file, index) => {
+              const result = results.find(r => r.file === file);
+              const isSelected = index === currentImageIndex;
+              const isProcessing = isAnalyzing && index === currentImageIndex;
+              
+              return (
+                <div
+                  key={`${file.name}-${index}`}
+                  className={`cursor-pointer transition-all duration-200 ${
+                    isSelected ? 'ring-2 ring-blue-400' : ''
+                  }`}
+                  onClick={() => handleImageSelect(index)}
+                >
+                  <GlassContainer
+                    bgColor={isSelected ? GLASS_EFFECTS.COLORS.BUTTON_BG : GLASS_EFFECTS.COLORS.DEFAULT_BG}
+                    className="p-3 rounded-lg"
+                  >
+                    <div className="flex items-start gap-3">
+                      {/* Thumbnail */}
+                      <div className="w-16 h-16 bg-gray-700 rounded flex items-center justify-center text-xs flex-shrink-0">
+                        <img
+                          src={URL.createObjectURL(file)}
+                          alt={file.name}
+                          className="w-full h-full object-cover rounded"
+                          onLoad={(e) => URL.revokeObjectURL((e.target as HTMLImageElement).src)}
+                        />
+                      </div>
+                      
+                      {/* Content */}
+                      <div className="flex-1 min-w-0">
+                        <div className="text-sm font-medium truncate mb-1">
+                          {file.name}
+                        </div>
+                        
+                        {isProcessing && (
+                          <div className="text-xs text-blue-400">
+                            Processing...
+                          </div>
+                        )}
+                        
+                        {result && (
+                          <div className="text-xs">
+                            {result.error ? (
+                              <div className="text-red-400">
+                                Error: {result.error}
+                              </div>
+                            ) : (
+                              <div className="text-white/80">
+                                {result.caption}
+                              </div>
+                            )}
+                          </div>
+                        )}
+                      </div>
+                    </div>
+                  </GlassContainer>
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      </div>
+
+      {/* Draggable Prompt Input - Bottom Left */}
+      <DraggableContainer initialPosition="bottom-left">
+        <PromptInput 
+          onPromptChange={handlePromptChange}
+          disabled={isAnalyzing}
+        />
+      </DraggableContainer>
+    </div>
+  );
+}

src/components/ImageUpload.tsx

@@ -0,0 +1,129 @@
+import { useState, useCallback, useRef } from "react";
+import GlassButton from "./GlassButton";
+import GlassContainer from "./GlassContainer";
+import { GLASS_EFFECTS } from "../constants";
+
+interface ImageUploadProps {
+  onImagesUploaded: (files: File[]) => void;
+  isAnalyzing: boolean;
+}
+
+export default function ImageUpload({ onImagesUploaded, isAnalyzing }: ImageUploadProps) {
+  const [dragActive, setDragActive] = useState(false);
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFiles = useCallback(
+    (files: FileList | null) => {
+      if (!files) return;
+      
+      const imageFiles = Array.from(files).filter(file =>
+        file.type.startsWith("image/")
+      );
+      
+      if (imageFiles.length > 0) {
+        onImagesUploaded(imageFiles);
+      }
+    },
+    [onImagesUploaded]
+  );
+
+  const handleDrag = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+  }, []);
+
+  const handleDragIn = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    if (e.dataTransfer?.items && e.dataTransfer.items.length > 0) {
+      setDragActive(true);
+    }
+  }, []);
+
+  const handleDragOut = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setDragActive(false);
+  }, []);
+
+  const handleDrop = useCallback(
+    (e: React.DragEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      setDragActive(false);
+      
+      if (e.dataTransfer?.files && e.dataTransfer.files.length > 0) {
+        handleFiles(e.dataTransfer.files);
+      }
+    },
+    [handleFiles]
+  );
+
+  const handleFileInputChange = useCallback(
+    (e: React.ChangeEvent<HTMLInputElement>) => {
+      handleFiles(e.target.files);
+    },
+    [handleFiles]
+  );
+
+  const handleClick = useCallback(() => {
+    if (!isAnalyzing) {
+      fileInputRef.current?.click();
+    }
+  }, [isAnalyzing]);
+
+  return (
+    <div className="absolute inset-0 flex items-center justify-center">
+      <GlassContainer
+        bgColor={dragActive ? GLASS_EFFECTS.COLORS.BUTTON_BG : GLASS_EFFECTS.COLORS.DEFAULT_BG}
+        className={`p-8 rounded-2xl border-2 border-dashed transition-all duration-300 cursor-pointer max-w-md mx-4 ${
+          dragActive ? "border-blue-400 scale-105" : "border-white/30"
+        } ${isAnalyzing ? "opacity-50 pointer-events-none" : "hover:border-white/50"}`}
+        onDragEnter={handleDragIn}
+        onDragLeave={handleDragOut}
+        onDragOver={handleDrag}
+        onDrop={handleDrop}
+        onClick={handleClick}
+      >
+        <div className="text-center text-white">
+          <div className="mb-4">
+            <svg
+              className="mx-auto w-16 h-16 text-white/60"
+              fill="none"
+              stroke="currentColor"
+              viewBox="0 0 24 24"
+            >
+              <path
+                strokeLinecap="round"
+                strokeLinejoin="round"
+                strokeWidth={1.5}
+                d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"
+              />
+            </svg>
+          </div>
+          
+          <h3 className="text-xl font-semibold mb-2">Upload Images</h3>
+          <p className="text-white/80 mb-4">
+            Drag and drop images here, or click to select files
+          </p>
+          <p className="text-sm text-white/60 mb-6">
+            Supports JPG, PNG, GIF, WebP formats. Multiple files allowed.
+          </p>
+          
+          <GlassButton disabled={isAnalyzing}>
+            {isAnalyzing ? "Analyzing..." : "Choose Files"}
+          </GlassButton>
+        </div>
+        
+        <input
+          ref={fileInputRef}
+          type="file"
+          multiple
+          accept="image/*"
+          onChange={handleFileInputChange}
+          className="hidden"
+        />
+      </GlassContainer>
+    </div>
+  );
+}

src/components/PromptInput.tsx

@@ -5,9 +5,10 @@ import GlassContainer from "./GlassContainer";
 interface PromptInputProps {
   onPromptChange: (prompt: string) => void;
   defaultPrompt?: string;
+  disabled?: boolean;
 }
 
-export default function PromptInput({ onPromptChange, defaultPrompt = PROMPTS.default }: PromptInputProps) {
+export default function PromptInput({ onPromptChange, defaultPrompt = PROMPTS.default, disabled = false }: PromptInputProps) {
   const [prompt, setPrompt] = useState(defaultPrompt);
   const [showSuggestions, setShowSuggestions] = useState(false);
   const inputRef = useRef<HTMLTextAreaElement>(null);
@@ -116,10 +117,13 @@ export default function PromptInput({ onPromptChange, defaultPrompt = PROMPTS.de
                 ref={inputRef}
                 value={prompt}
                 onChange={handleInputChange}
-                onFocus={handleInputFocus}
-                onBlur={handleInputBlur}
-                onClick={handleInputClick}
-                className="search-input w-full py-3 pl-4 pr-8 rounded-xl text-white text-base transition-all duration-400 border resize-none focus:outline-none focus:-translate-y-0.5 focus:shadow-lg"
+                onFocus={disabled ? undefined : handleInputFocus}
+                onBlur={disabled ? undefined : handleInputBlur}
+                onClick={disabled ? undefined : handleInputClick}
+                disabled={disabled}
+                className={`search-input w-full py-3 pl-4 pr-8 rounded-xl text-white text-base transition-all duration-400 border resize-none focus:outline-none focus:-translate-y-0.5 focus:shadow-lg ${
+                  disabled ? 'opacity-50 cursor-not-allowed' : ''
+                }`}
                 style={{
                   background: "var(--input-bg)",
                   borderColor: "var(--input-border)",
@@ -132,7 +136,7 @@ export default function PromptInput({ onPromptChange, defaultPrompt = PROMPTS.de
                 placeholder={PROMPTS.placeholder}
                 rows={1}
               />
-              {prompt && (
+              {prompt && !disabled && (
                 <button
                   type="button"
                   onClick={clearInput}

src/context/VLMContext.tsx

@@ -67,9 +67,9 @@ export const VLMProvider: React.FC<React.PropsWithChildren> = ({ children }) =>
   );
 
   const runInference = useCallback(
-    async (video: HTMLVideoElement, instruction: string, onTextUpdate?: (text: string) => void): Promise<string> => {
+    async (imageSource: HTMLVideoElement | File, instruction: string, onTextUpdate?: (text: string) => void): Promise<string> => {
       if (inferenceLock.current) {
-        console.log("Inference already running, skipping frame");
+        console.log("Inference already running, skipping");
         return ""; // Return empty string to signal a skip
       }
       inferenceLock.current = true;
@@ -78,21 +78,35 @@ export const VLMProvider: React.FC<React.PropsWithChildren> = ({ children }) =>
         throw new Error("Model/processor not loaded");
       }
 
-      if (!canvasRef.current) {
-        canvasRef.current = document.createElement("canvas");
-      }
-      const canvas = canvasRef.current;
+      let rawImg: RawImage;
 
-      canvas.width = video.videoWidth;
-      canvas.height = video.videoHeight;
+      if (imageSource instanceof File) {
+        // Handle uploaded image file
+        const url = URL.createObjectURL(imageSource);
+        try {
+          rawImg = await RawImage.fromURL(url);
+        } finally {
+          URL.revokeObjectURL(url);
+        }
+      } else {
+        // Handle video frame (original logic)
+        if (!canvasRef.current) {
+          canvasRef.current = document.createElement("canvas");
+        }
+        const canvas = canvasRef.current;
+        const video = imageSource;
+
+        canvas.width = video.videoWidth;
+        canvas.height = video.videoHeight;
 
-      const ctx = canvas.getContext("2d", { willReadFrequently: true });
-      if (!ctx) throw new Error("Could not get canvas context");
+        const ctx = canvas.getContext("2d", { willReadFrequently: true });
+        if (!ctx) throw new Error("Could not get canvas context");
 
-      ctx.drawImage(video, 0, 0);
+        ctx.drawImage(video, 0, 0);
 
-      const frame = ctx.getImageData(0, 0, canvas.width, canvas.height);
-      const rawImg = new RawImage(frame.data, frame.width, frame.height, 4);
+        const frame = ctx.getImageData(0, 0, canvas.width, canvas.height);
+        rawImg = new RawImage(frame.data, frame.width, frame.height, 4);
+      }
       const messages = [
         {
           role: "system",

src/types/index.ts

@@ -1,4 +1,4 @@
-export type AppState = "requesting-permission" | "welcome" | "loading" | "captioning";
+export type AppState = "upload" | "loading" | "analyzing";
 
 export interface GlassEffectProps {
   baseFrequency?: number;
@@ -25,3 +25,9 @@ export interface Dimensions {
 }
 
 export type InitialPosition = "bottom-left" | "bottom-right" | Position;
+
+export interface ImageAnalysisResult {
+  file: File;
+  caption: string;
+  error?: string;
+}

src/types/vlm.ts

@@ -4,7 +4,7 @@ export type VLMContextValue = {
   error: string | null;
   loadModel: (onProgress?: (msg: string) => void) => Promise<void>;
   runInference: (
-    video: HTMLVideoElement,
+    imageSource: HTMLVideoElement | File,
     instruction: string,
     onTextUpdate?: (text: string) => void,
   ) => Promise<string>;