import { ArrowBack, CameraAlt, CheckCircle, Close, DocumentScanner, PhotoCamera, Replay, TextSnippet, } from '@mui/icons-material' import { Box, Button, CircularProgress, IconButton, LinearProgress, Typography, } from '@mui/joy' import { useCallback, useEffect, useRef, useState } from 'react' import { useDocumentScanner } from '../../hooks/useDocumentScanner' import { useResponsiveModal } from '../../hooks/useResponsiveModal' import { localAIService } from '../../service/LocalAIService' const SYSTEM_PROMPT = `You are helping create tasks for a household task management app. Given OCR text extracted from a photo, identify the most useful task a person should add to their task list. The task title should always start with an action verb when possible. Examples: Bill -> "Pay water bill" Appointment -> "Attend eye doctor appointment" Invitation -> "RSVP for wedding" Renewal Notice -> "Renew vehicle registration" Package Notice -> "Pick up package" School Form -> "Complete school permission form" Action Priority Rules: 1. Payments and bills 2. Deadlines and renewals 3. Appointments 4. Required forms 5. Informational actions (view, read, review) Rules: Generate at most one task. Focus on the most important action. Extract due dates and deadlines. Use appointment dates as due dates when appropriate. Do not invent information. If the content contains no actionable item, return null values. Include any important ID or URL or instructions in the description Titles must be specific and useful at a glance. Include the organization, provider, event, or subject when available. Avoid generic document names. Return valid JSON only. Output: { "taskName": string | null, "description": string | null, "dueDate": string | null, "confidence": number }` async function runNativeOCR(imageSource) { const { Ocr } = await import('@jcesarmobile/capacitor-ocr') // Convert Capacitor WebView file URL → native file:// URL the plugin can read const image = imageSource.includes('/_capacitor_file_/') ? 'file://' + imageSource.replace(/^https?:\/\/localhost\/_capacitor_file_/, '') : imageSource const result = await Ocr.process({ image }) return result.results.map(r => r.text).join('\n').trim() } async function runOCR(imageSource, onProgress) { const { createWorker } = await import('tesseract.js') const worker = await createWorker('eng', 1, { logger: m => { if (m.status === 'recognizing text' && onProgress) { onProgress(Math.round(m.progress * 100)) } }, }) const { data } = await worker.recognize(imageSource) await worker.terminate() return data.text?.trim() || '' } async function extractTaskFromOCR(ocrText) { const messages = [ { role: 'system', content: SYSTEM_PROMPT }, { role: 'user', content: `OCR Text:\n${ocrText}` }, ] const result = await localAIService.plainChat(messages) if (!result) return null const jsonMatch = result.match(/\{[\s\S]*\}/) if (!jsonMatch) return null try { return JSON.parse(jsonMatch[0]) } catch { return null } } const PhotoTaskModal = ({ open, onClose, onTaskExtracted }) => { const { ResponsiveModal } = useResponsiveModal() const { isNativeScanner, scanDocument } = useDocumentScanner() const videoRef = useRef(null) const canvasRef = useRef(null) const streamRef = useRef(null) const fileInputRef = useRef(null) const [phase, setPhase] = useState('capture') // capture | preview | ocr | llm | done | error const [capturedImage, setCapturedImage] = useState(null) const [ocrProgress, setOcrProgress] = useState(0) const [ocrText, setOcrText] = useState('') const [ocrMethod, setOcrMethod] = useState('tesseract') const [showRawText, setShowRawText] = useState(false) const [taskResult, setTaskResult] = useState(null) const [errorMsg, setErrorMsg] = useState('') const [cameraAvailable, setCameraAvailable] = useState(true) const startCamera = useCallback(async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment' }, }) streamRef.current = stream if (videoRef.current) { videoRef.current.srcObject = stream } setCameraAvailable(true) } catch { setCameraAvailable(false) } }, []) const stopCamera = useCallback(() => { if (streamRef.current) { streamRef.current.getTracks().forEach(t => t.stop()) streamRef.current = null } }, []) useEffect(() => { if (open && phase === 'capture' && !isNativeScanner) { startCamera() } return () => { stopCamera() } }, [open, phase, isNativeScanner, startCamera, stopCamera]) const handleCapture = () => { if (!videoRef.current || !canvasRef.current) return const video = videoRef.current const canvas = canvasRef.current canvas.width = video.videoWidth canvas.height = video.videoHeight canvas.getContext('2d').drawImage(video, 0, 0) const dataUrl = canvas.toDataURL('image/jpeg', 0.9) setCapturedImage(dataUrl) stopCamera() setPhase('preview') } const handleFileSelect = e => { const file = e.target.files?.[0] if (!file) return const reader = new FileReader() reader.onload = ev => { setCapturedImage(ev.target.result) stopCamera() setPhase('preview') } reader.readAsDataURL(file) } const handleBackToPreview = () => { setOcrText('') setTaskResult(null) setErrorMsg('') setOcrProgress(0) setShowRawText(false) setPhase('preview') } const handleProcess = async (method = 'tesseract') => { setOcrMethod(method) setPhase('ocr') setOcrProgress(0) setErrorMsg('') setShowRawText(false) try { let text if (method === 'native') { try { text = await runNativeOCR(capturedImage) } catch { throw new Error('Native OCR is only available on iOS and Android devices.') } } else { text = await runOCR(capturedImage, pct => setOcrProgress(pct)) } setOcrText(text) if (!text) { setErrorMsg('No text found in the image. Please try a clearer photo.') setPhase('error') return } setPhase('llm') const task = await extractTaskFromOCR(text) if (!task || !task.taskName) { setErrorMsg('Could not identify a task from this image. Please try a different photo.') setPhase('error') return } setTaskResult(task) setPhase('done') } catch (e) { setErrorMsg(`Processing failed: ${e.message || 'Unknown error'}`) setPhase('error') } } const handleRetake = () => { setCapturedImage(null) setOcrText('') setTaskResult(null) setErrorMsg('') setOcrProgress(0) setShowRawText(false) setPhase('capture') } const handleNativeScan = async () => { const { image, cancelled, error } = await scanDocument() if (cancelled) return if (error || !image) { setErrorMsg(error ? `Scanner error: ${error}` : 'Scan cancelled or failed.') setPhase('error') return } setCapturedImage(image) stopCamera() setPhase('preview') } const handleConfirm = () => { if (taskResult) { onTaskExtracted(taskResult) } handleClose() } const handleClose = () => { stopCamera() setCapturedImage(null) setOcrText('') setTaskResult(null) setErrorMsg('') setOcrProgress(0) setShowRawText(false) setPhase('capture') onClose() } const isProcessing = phase === 'ocr' || phase === 'llm' return ( {(phase === 'capture' || phase === 'preview') && ( {phase === 'capture' && !isNativeScanner && cameraAvailable && ( )} {isProcessing && ( {capturedImage && ( Processing )} {phase === 'ocr' && ocrMethod === 'tesseract' && ( <> Reading text from image… {ocrProgress}% )} {phase === 'ocr' && ocrMethod === 'native' && ( Running native OCR… )} {phase === 'llm' && ( Identifying task with AI… )} )} {phase === 'done' && taskResult && ( Task identified {taskResult.taskName} {taskResult.description && ( {taskResult.description} )} {taskResult.dueDate && ( Due: {taskResult.dueDate} )} {ocrText && ( <> {showRawText && ( {ocrText} )} )} )} {phase === 'error' && ( {errorMsg} )} {phase === 'capture' && ( <> {isNativeScanner ? ( ) : ( cameraAvailable && ( ) )} )} {phase === 'preview' && ( <> {isNativeScanner && ( )} )} {phase === 'error' && ( <> )} {phase === 'done' && ( <> )} {!isProcessing && ( )} ) } export default PhotoTaskModal