Add Ability to turnOCR photo then create task (Merge support-local-ai)
This commit is contained in:
50
src/hooks/useDocumentScanner.js
Normal file
50
src/hooks/useDocumentScanner.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { Capacitor } from '@capacitor/core'
|
||||
|
||||
/**
|
||||
* Normalizes a raw image string from the native document scanner into a
|
||||
* format that can be used as an <img> src and passed to Tesseract.js.
|
||||
*
|
||||
* Android returns file:// or absolute paths → convert via Capacitor.convertFileSrc
|
||||
* iOS returns raw base64 (no data: prefix) → prepend the data URI scheme
|
||||
*/
|
||||
function normalizeScannedImage(raw) {
|
||||
if (!raw) return null
|
||||
if (raw.startsWith('data:')) return raw
|
||||
if (raw.startsWith('http://') || raw.startsWith('https://') || raw.startsWith('content://')) return raw
|
||||
if (raw.startsWith('/') || raw.startsWith('file://')) return Capacitor.convertFileSrc(raw)
|
||||
// iOS base64 without prefix
|
||||
return `data:image/jpeg;base64,${raw}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for native document scanning via @capgo/capacitor-document-scanner.
|
||||
*
|
||||
* On native: opens the OS document scanner (edge detection, perspective correction).
|
||||
* On web: `scanDocument` returns null — callers should fall back to their own camera UI.
|
||||
*/
|
||||
export function useDocumentScanner() {
|
||||
const isNativeScanner = Capacitor.isNativePlatform()
|
||||
|
||||
const scanDocument = async ({ maxDocuments = 1, quality = 90, letUserAdjustCrop = true } = {}) => {
|
||||
if (!isNativeScanner) return { image: null, cancelled: false }
|
||||
|
||||
try {
|
||||
const { DocumentScanner } = await import('@capgo/capacitor-document-scanner')
|
||||
const { scannedImages } = await DocumentScanner.scanDocument({
|
||||
croppedImageQuality: quality,
|
||||
maxNumDocuments: maxDocuments,
|
||||
letUserAdjustCrop,
|
||||
})
|
||||
|
||||
if (!scannedImages?.length) return { image: null, cancelled: true }
|
||||
|
||||
const normalized = normalizeScannedImage(scannedImages[0])
|
||||
return { image: normalized, cancelled: false }
|
||||
} catch (e) {
|
||||
console.error('[DocumentScanner] scan failed:', e)
|
||||
return { image: null, cancelled: false, error: e.message }
|
||||
}
|
||||
}
|
||||
|
||||
return { isNativeScanner, scanDocument }
|
||||
}
|
||||
72
src/service/AIPromptCache.js
Normal file
72
src/service/AIPromptCache.js
Normal file
@@ -0,0 +1,72 @@
|
||||
const ENABLED_KEY = 'ai_prompt_cache_enabled'
|
||||
const ENTRY_PREFIX = 'ai_prompt_cache_'
|
||||
const INDEX_KEY = 'ai_prompt_cache_index'
|
||||
|
||||
function djb2(str) {
|
||||
let hash = 5381
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash = ((hash << 5) + hash) ^ str.charCodeAt(i)
|
||||
hash = hash >>> 0
|
||||
}
|
||||
return hash.toString(36)
|
||||
}
|
||||
|
||||
export function isCacheEnabled() {
|
||||
try {
|
||||
return localStorage.getItem(ENABLED_KEY) === 'true'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export function setCacheEnabled(enabled) {
|
||||
try {
|
||||
localStorage.setItem(ENABLED_KEY, String(enabled))
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export function hashContent(content) {
|
||||
return djb2(typeof content === 'string' ? content : JSON.stringify(content))
|
||||
}
|
||||
|
||||
function getIndex() {
|
||||
try {
|
||||
return JSON.parse(localStorage.getItem(INDEX_KEY) || '[]')
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export function getCached(hash) {
|
||||
if (!isCacheEnabled()) return null
|
||||
try {
|
||||
const raw = localStorage.getItem(ENTRY_PREFIX + hash)
|
||||
return raw ? JSON.parse(raw) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function setCached(hash, value) {
|
||||
if (!isCacheEnabled()) return
|
||||
try {
|
||||
localStorage.setItem(ENTRY_PREFIX + hash, JSON.stringify(value))
|
||||
const index = getIndex()
|
||||
if (!index.includes(hash)) {
|
||||
index.push(hash)
|
||||
localStorage.setItem(INDEX_KEY, JSON.stringify(index))
|
||||
}
|
||||
} catch { /* storage full, ignore */ }
|
||||
}
|
||||
|
||||
export function getCacheStats() {
|
||||
return { count: getIndex().length }
|
||||
}
|
||||
|
||||
export function clearCache() {
|
||||
const index = getIndex()
|
||||
index.forEach(h => {
|
||||
try { localStorage.removeItem(ENTRY_PREFIX + h) } catch { /* ignore */ }
|
||||
})
|
||||
try { localStorage.removeItem(INDEX_KEY) } catch { /* ignore */ }
|
||||
}
|
||||
141
src/service/LocalAIService.js
Normal file
141
src/service/LocalAIService.js
Normal file
@@ -0,0 +1,141 @@
|
||||
import { Capacitor } from '@capacitor/core'
|
||||
import { getCached, hashContent, setCached } from './AIPromptCache'
|
||||
|
||||
// Native-only local AI service using @capacitor/local-llm.
|
||||
// On web, all methods return 'unavailable' / null — no WebLLM.
|
||||
class LocalAIService {
|
||||
constructor() {
|
||||
this._availability = null
|
||||
this._sessionId = 'donetick-summary'
|
||||
this._warmedUp = false
|
||||
}
|
||||
|
||||
get isNative() {
|
||||
return Capacitor.isNativePlatform()
|
||||
}
|
||||
|
||||
async checkAvailability() {
|
||||
if (!this.isNative) {
|
||||
this._availability = 'unavailable'
|
||||
return 'unavailable'
|
||||
}
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
const { status } = await LocalLLM.systemAvailability()
|
||||
this._availability = status
|
||||
return status
|
||||
} catch (e) {
|
||||
this._availability = 'unavailable'
|
||||
return 'unavailable'
|
||||
}
|
||||
}
|
||||
|
||||
async getStatus() {
|
||||
if (this._availability !== null) return this._availability
|
||||
return this.checkAvailability()
|
||||
}
|
||||
|
||||
async isAvailable() {
|
||||
return (await this.getStatus()) === 'available'
|
||||
}
|
||||
|
||||
resetAvailability() {
|
||||
this._availability = null
|
||||
}
|
||||
|
||||
async download(onStatusChange) {
|
||||
if (!this.isNative) return
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
if (onStatusChange) {
|
||||
LocalLLM.addListener('systemAvailabilityChange', ({ status }) => {
|
||||
this._availability = status
|
||||
onStatusChange(status)
|
||||
})
|
||||
}
|
||||
await LocalLLM.download()
|
||||
} catch {
|
||||
// download not available on iOS, ignore
|
||||
}
|
||||
}
|
||||
|
||||
async warmup() {
|
||||
if (this._warmedUp || !this.isNative) return
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
await LocalLLM.warmup({ sessionId: this._sessionId })
|
||||
this._warmedUp = true
|
||||
} catch {
|
||||
// non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
async _nativePrompt(text) {
|
||||
await this.warmup()
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
const { text: out } = await LocalLLM.prompt({ prompt: text, sessionId: this._sessionId })
|
||||
return out?.trim() || null
|
||||
} finally {
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
await LocalLLM.endSession({ sessionId: this._sessionId })
|
||||
this._warmedUp = false
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
// Plain chat — no tools. Returns answer string or null.
|
||||
async plainChat(messages) {
|
||||
const available = await this.isAvailable()
|
||||
if (!available) return null
|
||||
|
||||
const cacheHash = hashContent(['plain', ...messages])
|
||||
const cached = getCached(cacheHash)
|
||||
if (cached) return cached
|
||||
|
||||
if (!this.isNative) return null
|
||||
|
||||
try {
|
||||
const systemMsg = messages.find(m => m.role === 'system')?.content || ''
|
||||
const userMsg = messages.find(m => m.role === 'user')?.content || ''
|
||||
const result = await this._nativePrompt(`${systemMsg}\n\nUser: ${userMsg}\nAssistant:`)
|
||||
if (result) setCached(cacheHash, result)
|
||||
return result
|
||||
} catch (e) {
|
||||
console.error('[LocalAI] plainChat() failed:', e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the summary string or null if LLM is unavailable
|
||||
async summarize(prompt) {
|
||||
const available = await this.isAvailable()
|
||||
if (!available) return null
|
||||
|
||||
const cacheHash = hashContent(prompt)
|
||||
const cached = getCached(cacheHash)
|
||||
if (cached) return cached
|
||||
|
||||
if (!this.isNative) return null
|
||||
|
||||
try {
|
||||
await this.warmup()
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
const { text } = await LocalLLM.prompt({ prompt, sessionId: this._sessionId })
|
||||
const result = text?.trim() || null
|
||||
if (result) setCached(cacheHash, result)
|
||||
return result
|
||||
} catch {
|
||||
return null
|
||||
} finally {
|
||||
try {
|
||||
const { LocalLLM } = await import('@capacitor/local-llm')
|
||||
await LocalLLM.endSession({ sessionId: this._sessionId })
|
||||
this._warmedUp = false
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const localAIService = new LocalAIService()
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Add } from '@mui/icons-material'
|
||||
import { Add, CameraAlt } from '@mui/icons-material'
|
||||
import { Box, Button, Typography } from '@mui/joy'
|
||||
import { useMediaQuery } from '@mui/material'
|
||||
import * as chrono from 'chrono-node'
|
||||
@@ -28,6 +28,7 @@ import DueDatePickerField from './DueDatePickerField'
|
||||
import LabelsPickerField from './LabelsPickerField'
|
||||
import LearnMoreButton from './LearnMore'
|
||||
import NotificationPickerField from './NotificationPickerField'
|
||||
import PhotoTaskModal from './PhotoTaskModal'
|
||||
import PriorityPickerField from './PriorityPickerField'
|
||||
import RepeatPickerField from './RepeatPickerField'
|
||||
import RichTextEditor from './RichTextEditor'
|
||||
@@ -103,6 +104,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
const [showKeyboardShortcuts, setShowKeyboardShortcuts] = useState(false)
|
||||
const [projectId, setProjectId] = useState(getInitialProject())
|
||||
const [attachments, setAttachments] = useState([])
|
||||
const [photoModalOpen, setPhotoModalOpen] = useState(false)
|
||||
|
||||
// Priority colors
|
||||
const priorityColors = {
|
||||
@@ -543,6 +545,23 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
createChore()
|
||||
}
|
||||
|
||||
const handleTaskExtracted = ({ taskName, description: extractedDesc, dueDate: extractedDue }) => {
|
||||
if (taskName) {
|
||||
processText(taskName)
|
||||
}
|
||||
if (extractedDesc) {
|
||||
setDescription(extractedDesc)
|
||||
setHasDescription(true)
|
||||
}
|
||||
if (extractedDue) {
|
||||
const m = moment(new Date(extractedDue))
|
||||
if (m.isValid()) {
|
||||
setDueDateOnly(m.format('YYYY-MM-DD'))
|
||||
setDueDate(m.endOf('day').format('YYYY-MM-DDTHH:mm:ss'))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleCloseModal = forceRefetch => {
|
||||
onClose(forceRefetch)
|
||||
setTaskText('')
|
||||
@@ -664,6 +683,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<ResponsiveModal
|
||||
open={isModalOpen}
|
||||
onClose={handleCloseModal}
|
||||
@@ -766,6 +786,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
placeholder='Type your task...'
|
||||
onChange={text => {
|
||||
setTaskText(text)
|
||||
if (!text) setTaskTitle('')
|
||||
}}
|
||||
customRenderer={renderedParts}
|
||||
onEnterPressed={handleEnterPressed}
|
||||
@@ -909,6 +930,14 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
Description
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
startDecorator={<CameraAlt />}
|
||||
variant='plain'
|
||||
size='sm'
|
||||
onClick={() => setPhotoModalOpen(true)}
|
||||
>
|
||||
Scan Photo
|
||||
</Button>
|
||||
|
||||
{!hasSubTasks && (
|
||||
<Button
|
||||
@@ -935,6 +964,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
<RichTextEditor
|
||||
ref={richTextEditorRef}
|
||||
onChange={setDescription}
|
||||
value={description || ''}
|
||||
entityType={'chore_description'}
|
||||
/>
|
||||
</div>
|
||||
@@ -952,6 +982,12 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
|
||||
</Box>
|
||||
)}
|
||||
</ResponsiveModal>
|
||||
<PhotoTaskModal
|
||||
open={photoModalOpen}
|
||||
onClose={() => setPhotoModalOpen(false)}
|
||||
onTaskExtracted={handleTaskExtracted}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
595
src/views/components/PhotoTaskModal.jsx
Normal file
595
src/views/components/PhotoTaskModal.jsx
Normal file
@@ -0,0 +1,595 @@
|
||||
import {
|
||||
ArrowBack,
|
||||
CameraAlt,
|
||||
CheckCircle,
|
||||
Close,
|
||||
DocumentScanner,
|
||||
PhotoCamera,
|
||||
Replay,
|
||||
TextSnippet,
|
||||
} from '@mui/icons-material'
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
CircularProgress,
|
||||
IconButton,
|
||||
LinearProgress,
|
||||
Typography,
|
||||
} from '@mui/joy'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { useDocumentScanner } from '../../hooks/useDocumentScanner'
|
||||
import { useResponsiveModal } from '../../hooks/useResponsiveModal'
|
||||
import { localAIService } from '../../service/LocalAIService'
|
||||
|
||||
const SYSTEM_PROMPT = `You are helping create tasks for a household task management app.
|
||||
|
||||
Given OCR text extracted from a photo, identify the most useful task a person should add to their task list.
|
||||
|
||||
The task title should always start with an action verb when possible.
|
||||
|
||||
Examples:
|
||||
|
||||
Bill -> "Pay water bill"
|
||||
Appointment -> "Attend eye doctor appointment"
|
||||
Invitation -> "RSVP for wedding"
|
||||
Renewal Notice -> "Renew vehicle registration"
|
||||
Package Notice -> "Pick up package"
|
||||
School Form -> "Complete school permission form"
|
||||
|
||||
Action Priority Rules:
|
||||
|
||||
1. Payments and bills
|
||||
2. Deadlines and renewals
|
||||
3. Appointments
|
||||
4. Required forms
|
||||
5. Informational actions (view, read, review)
|
||||
|
||||
|
||||
Rules:
|
||||
|
||||
Generate at most one task.
|
||||
Focus on the most important action.
|
||||
Extract due dates and deadlines.
|
||||
Use appointment dates as due dates when appropriate.
|
||||
Do not invent information.
|
||||
If the content contains no actionable item, return null values.
|
||||
Include any important ID or URL or instructions in the description
|
||||
Titles must be specific and useful at a glance.
|
||||
Include the organization, provider, event, or subject when available.
|
||||
Avoid generic document names.
|
||||
Return valid JSON only.
|
||||
Output:
|
||||
|
||||
{
|
||||
"taskName": string | null,
|
||||
"description": string | null,
|
||||
"dueDate": string | null,
|
||||
"confidence": number
|
||||
}`
|
||||
|
||||
async function runNativeOCR(imageSource) {
|
||||
const { Ocr } = await import('@jcesarmobile/capacitor-ocr')
|
||||
|
||||
// Convert Capacitor WebView file URL → native file:// URL the plugin can read
|
||||
const image = imageSource.includes('/_capacitor_file_/')
|
||||
? 'file://' + imageSource.replace(/^https?:\/\/localhost\/_capacitor_file_/, '')
|
||||
: imageSource
|
||||
|
||||
const result = await Ocr.process({ image })
|
||||
return result.results.map(r => r.text).join('\n').trim()
|
||||
}
|
||||
|
||||
async function runOCR(imageSource, onProgress) {
|
||||
const { createWorker } = await import('tesseract.js')
|
||||
const worker = await createWorker('eng', 1, {
|
||||
logger: m => {
|
||||
if (m.status === 'recognizing text' && onProgress) {
|
||||
onProgress(Math.round(m.progress * 100))
|
||||
}
|
||||
},
|
||||
})
|
||||
const { data } = await worker.recognize(imageSource)
|
||||
await worker.terminate()
|
||||
return data.text?.trim() || ''
|
||||
}
|
||||
|
||||
async function extractTaskFromOCR(ocrText) {
|
||||
const messages = [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: `OCR Text:\n${ocrText}` },
|
||||
]
|
||||
|
||||
const result = await localAIService.plainChat(messages)
|
||||
if (!result) return null
|
||||
|
||||
const jsonMatch = result.match(/\{[\s\S]*\}/)
|
||||
if (!jsonMatch) return null
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonMatch[0])
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const PhotoTaskModal = ({ open, onClose, onTaskExtracted }) => {
|
||||
const { ResponsiveModal } = useResponsiveModal()
|
||||
const { isNativeScanner, scanDocument } = useDocumentScanner()
|
||||
const videoRef = useRef(null)
|
||||
const canvasRef = useRef(null)
|
||||
const streamRef = useRef(null)
|
||||
const fileInputRef = useRef(null)
|
||||
|
||||
const [phase, setPhase] = useState('capture') // capture | preview | ocr | llm | done | error
|
||||
const [capturedImage, setCapturedImage] = useState(null)
|
||||
const [ocrProgress, setOcrProgress] = useState(0)
|
||||
const [ocrText, setOcrText] = useState('')
|
||||
const [ocrMethod, setOcrMethod] = useState('tesseract')
|
||||
const [showRawText, setShowRawText] = useState(false)
|
||||
const [taskResult, setTaskResult] = useState(null)
|
||||
const [errorMsg, setErrorMsg] = useState('')
|
||||
const [cameraAvailable, setCameraAvailable] = useState(true)
|
||||
|
||||
const startCamera = useCallback(async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode: 'environment' },
|
||||
})
|
||||
streamRef.current = stream
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = stream
|
||||
}
|
||||
setCameraAvailable(true)
|
||||
} catch {
|
||||
setCameraAvailable(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach(t => t.stop())
|
||||
streamRef.current = null
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (open && phase === 'capture' && !isNativeScanner) {
|
||||
startCamera()
|
||||
}
|
||||
return () => {
|
||||
stopCamera()
|
||||
}
|
||||
}, [open, phase, isNativeScanner, startCamera, stopCamera])
|
||||
|
||||
const handleCapture = () => {
|
||||
if (!videoRef.current || !canvasRef.current) return
|
||||
const video = videoRef.current
|
||||
const canvas = canvasRef.current
|
||||
canvas.width = video.videoWidth
|
||||
canvas.height = video.videoHeight
|
||||
canvas.getContext('2d').drawImage(video, 0, 0)
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', 0.9)
|
||||
setCapturedImage(dataUrl)
|
||||
stopCamera()
|
||||
setPhase('preview')
|
||||
}
|
||||
|
||||
const handleFileSelect = e => {
|
||||
const file = e.target.files?.[0]
|
||||
if (!file) return
|
||||
const reader = new FileReader()
|
||||
reader.onload = ev => {
|
||||
setCapturedImage(ev.target.result)
|
||||
stopCamera()
|
||||
setPhase('preview')
|
||||
}
|
||||
reader.readAsDataURL(file)
|
||||
}
|
||||
|
||||
const handleBackToPreview = () => {
|
||||
setOcrText('')
|
||||
setTaskResult(null)
|
||||
setErrorMsg('')
|
||||
setOcrProgress(0)
|
||||
setShowRawText(false)
|
||||
setPhase('preview')
|
||||
}
|
||||
|
||||
const handleProcess = async (method = 'tesseract') => {
|
||||
setOcrMethod(method)
|
||||
setPhase('ocr')
|
||||
setOcrProgress(0)
|
||||
setErrorMsg('')
|
||||
setShowRawText(false)
|
||||
|
||||
try {
|
||||
let text
|
||||
if (method === 'native') {
|
||||
try {
|
||||
text = await runNativeOCR(capturedImage)
|
||||
} catch {
|
||||
throw new Error('Native OCR is only available on iOS and Android devices.')
|
||||
}
|
||||
} else {
|
||||
text = await runOCR(capturedImage, pct => setOcrProgress(pct))
|
||||
}
|
||||
setOcrText(text)
|
||||
|
||||
if (!text) {
|
||||
setErrorMsg('No text found in the image. Please try a clearer photo.')
|
||||
setPhase('error')
|
||||
return
|
||||
}
|
||||
|
||||
setPhase('llm')
|
||||
const task = await extractTaskFromOCR(text)
|
||||
|
||||
if (!task || !task.taskName) {
|
||||
setErrorMsg('Could not identify a task from this image. Please try a different photo.')
|
||||
setPhase('error')
|
||||
return
|
||||
}
|
||||
|
||||
setTaskResult(task)
|
||||
setPhase('done')
|
||||
} catch (e) {
|
||||
setErrorMsg(`Processing failed: ${e.message || 'Unknown error'}`)
|
||||
setPhase('error')
|
||||
}
|
||||
}
|
||||
|
||||
const handleRetake = () => {
|
||||
setCapturedImage(null)
|
||||
setOcrText('')
|
||||
setTaskResult(null)
|
||||
setErrorMsg('')
|
||||
setOcrProgress(0)
|
||||
setShowRawText(false)
|
||||
setPhase('capture')
|
||||
}
|
||||
|
||||
const handleNativeScan = async () => {
|
||||
const { image, cancelled, error } = await scanDocument()
|
||||
if (cancelled) return
|
||||
if (error || !image) {
|
||||
setErrorMsg(error ? `Scanner error: ${error}` : 'Scan cancelled or failed.')
|
||||
setPhase('error')
|
||||
return
|
||||
}
|
||||
setCapturedImage(image)
|
||||
stopCamera()
|
||||
setPhase('preview')
|
||||
}
|
||||
|
||||
const handleConfirm = () => {
|
||||
if (taskResult) {
|
||||
onTaskExtracted(taskResult)
|
||||
}
|
||||
handleClose()
|
||||
}
|
||||
|
||||
const handleClose = () => {
|
||||
stopCamera()
|
||||
setCapturedImage(null)
|
||||
setOcrText('')
|
||||
setTaskResult(null)
|
||||
setErrorMsg('')
|
||||
setOcrProgress(0)
|
||||
setShowRawText(false)
|
||||
setPhase('capture')
|
||||
onClose()
|
||||
}
|
||||
|
||||
const isProcessing = phase === 'ocr' || phase === 'llm'
|
||||
|
||||
return (
|
||||
<ResponsiveModal
|
||||
open={open}
|
||||
onClose={handleClose}
|
||||
size='md'
|
||||
fullWidth
|
||||
title='Scan photo to create task'
|
||||
>
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
{(phase === 'capture' || phase === 'preview') && (
|
||||
<Box
|
||||
sx={{
|
||||
position: 'relative',
|
||||
width: '100%',
|
||||
borderRadius: 'md',
|
||||
overflow: 'hidden',
|
||||
bgcolor: 'background.level1',
|
||||
minHeight: 240,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
{phase === 'capture' && !isNativeScanner && cameraAvailable && (
|
||||
<video
|
||||
ref={videoRef}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
style={{ width: '100%', display: 'block' }}
|
||||
/>
|
||||
)}
|
||||
{phase === 'capture' && isNativeScanner && (
|
||||
<Box sx={{ textAlign: 'center', p: 4 }}>
|
||||
<DocumentScanner sx={{ fontSize: 64, opacity: 0.4, mb: 1 }} />
|
||||
<Typography level='body-sm' sx={{ opacity: 0.6 }}>
|
||||
Tap "Scan Document" to open the scanner
|
||||
</Typography>
|
||||
</Box>
|
||||
)}
|
||||
{phase === 'capture' && !isNativeScanner && !cameraAvailable && (
|
||||
<Box sx={{ textAlign: 'center', p: 3 }}>
|
||||
<CameraAlt sx={{ fontSize: 48, opacity: 0.5, mb: 1 }} />
|
||||
<Typography level='body-sm' sx={{ opacity: 0.7 }}>
|
||||
Camera not available
|
||||
</Typography>
|
||||
</Box>
|
||||
)}
|
||||
{phase === 'preview' && capturedImage && (
|
||||
<img
|
||||
src={capturedImage}
|
||||
alt='Captured document'
|
||||
style={{ width: '100%', display: 'block' }}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<canvas ref={canvasRef} style={{ display: 'none' }} />
|
||||
|
||||
{isProcessing && (
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
py: 4,
|
||||
}}
|
||||
>
|
||||
{capturedImage && (
|
||||
<img
|
||||
src={capturedImage}
|
||||
alt='Processing'
|
||||
style={{
|
||||
width: '100%',
|
||||
borderRadius: 8,
|
||||
opacity: 0.6,
|
||||
maxHeight: 200,
|
||||
objectFit: 'contain',
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<CircularProgress size='md' />
|
||||
{phase === 'ocr' && ocrMethod === 'tesseract' && (
|
||||
<>
|
||||
<Typography level='body-sm'>
|
||||
Reading text from image… {ocrProgress}%
|
||||
</Typography>
|
||||
<LinearProgress determinate value={ocrProgress} sx={{ width: '100%' }} />
|
||||
</>
|
||||
)}
|
||||
{phase === 'ocr' && ocrMethod === 'native' && (
|
||||
<Typography level='body-sm'>Running native OCR…</Typography>
|
||||
)}
|
||||
{phase === 'llm' && (
|
||||
<Typography level='body-sm'>Identifying task with AI…</Typography>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{phase === 'done' && taskResult && (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1.5 }}>
|
||||
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
|
||||
<CheckCircle color='success' />
|
||||
<Typography level='title-sm'>Task identified</Typography>
|
||||
</Box>
|
||||
<Box
|
||||
sx={{
|
||||
p: 1.5,
|
||||
borderRadius: 'md',
|
||||
bgcolor: 'background.level1',
|
||||
border: '1px solid',
|
||||
borderColor: 'divider',
|
||||
}}
|
||||
>
|
||||
<Typography level='title-sm'>{taskResult.taskName}</Typography>
|
||||
{taskResult.description && (
|
||||
<Typography level='body-xs' sx={{ mt: 0.5, opacity: 0.8 }}>
|
||||
{taskResult.description}
|
||||
</Typography>
|
||||
)}
|
||||
{taskResult.dueDate && (
|
||||
<Typography level='body-xs' sx={{ mt: 0.5, opacity: 0.7 }}>
|
||||
Due: {taskResult.dueDate}
|
||||
</Typography>
|
||||
)}
|
||||
</Box>
|
||||
{ocrText && (
|
||||
<>
|
||||
<Button
|
||||
size='sm'
|
||||
variant='plain'
|
||||
color='neutral'
|
||||
startDecorator={<TextSnippet />}
|
||||
onClick={() => setShowRawText(v => !v)}
|
||||
sx={{ alignSelf: 'flex-start' }}
|
||||
>
|
||||
{showRawText ? 'Hide Raw Text' : 'Show Raw Text'}
|
||||
</Button>
|
||||
{showRawText && (
|
||||
<Box
|
||||
sx={{
|
||||
p: 1.5,
|
||||
borderRadius: 'md',
|
||||
bgcolor: 'background.level2',
|
||||
border: '1px solid',
|
||||
borderColor: 'divider',
|
||||
maxHeight: 180,
|
||||
overflowY: 'auto',
|
||||
}}
|
||||
>
|
||||
<Typography
|
||||
level='body-xs'
|
||||
sx={{ whiteSpace: 'pre-wrap', fontFamily: 'monospace' }}
|
||||
>
|
||||
{ocrText}
|
||||
</Typography>
|
||||
</Box>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{phase === 'error' && (
|
||||
<Box
|
||||
sx={{
|
||||
p: 2,
|
||||
borderRadius: 'md',
|
||||
bgcolor: 'danger.softBg',
|
||||
color: 'danger.softColor',
|
||||
}}
|
||||
>
|
||||
<Typography level='body-sm'>{errorMsg}</Typography>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<Box sx={{ display: 'flex', gap: 1, justifyContent: 'flex-end' }}>
|
||||
{phase === 'capture' && (
|
||||
<>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<PhotoCamera />}
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
Upload Photo
|
||||
</Button>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type='file'
|
||||
accept='image/*'
|
||||
style={{ display: 'none' }}
|
||||
onChange={handleFileSelect}
|
||||
/>
|
||||
{isNativeScanner ? (
|
||||
<Button
|
||||
variant='solid'
|
||||
color='primary'
|
||||
startDecorator={<DocumentScanner />}
|
||||
onClick={handleNativeScan}
|
||||
>
|
||||
Scan Document
|
||||
</Button>
|
||||
) : (
|
||||
cameraAvailable && (
|
||||
<Button
|
||||
variant='solid'
|
||||
color='primary'
|
||||
startDecorator={<CameraAlt />}
|
||||
onClick={handleCapture}
|
||||
>
|
||||
Capture
|
||||
</Button>
|
||||
)
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{phase === 'preview' && (
|
||||
<>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<Replay />}
|
||||
onClick={handleRetake}
|
||||
>
|
||||
Retake
|
||||
</Button>
|
||||
{isNativeScanner && (
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='primary'
|
||||
onClick={() => handleProcess('native')}
|
||||
>
|
||||
Process Natively
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
variant='solid'
|
||||
color='primary'
|
||||
onClick={() => handleProcess('tesseract')}
|
||||
>
|
||||
Process Image
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{phase === 'error' && (
|
||||
<>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<ArrowBack />}
|
||||
onClick={handleBackToPreview}
|
||||
>
|
||||
Back
|
||||
</Button>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<Replay />}
|
||||
onClick={handleRetake}
|
||||
>
|
||||
Retake
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{phase === 'done' && (
|
||||
<>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<ArrowBack />}
|
||||
onClick={handleBackToPreview}
|
||||
>
|
||||
Back
|
||||
</Button>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color='neutral'
|
||||
startDecorator={<Replay />}
|
||||
onClick={handleRetake}
|
||||
>
|
||||
Retake
|
||||
</Button>
|
||||
<Button variant='solid' color='primary' onClick={handleConfirm}>
|
||||
Create Task
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{!isProcessing && (
|
||||
<IconButton
|
||||
variant='plain'
|
||||
color='neutral'
|
||||
onClick={handleClose}
|
||||
sx={{ ml: 'auto' }}
|
||||
>
|
||||
<Close />
|
||||
</IconButton>
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
</ResponsiveModal>
|
||||
)
|
||||
}
|
||||
|
||||
export default PhotoTaskModal
|
||||
Reference in New Issue
Block a user