Add Ability to turnOCR photo then create task (Merge support-local-ai)

This commit is contained in:
Mo Tarbin
2026-07-03 23:31:19 -04:00
parent dee41dad9e
commit 260087d1fa
12 changed files with 1890 additions and 510 deletions

View File

@@ -0,0 +1,50 @@
import { Capacitor } from '@capacitor/core'
/**
* Normalizes a raw image string from the native document scanner into a
* format that can be used as an <img> src and passed to Tesseract.js.
*
* Android returns file:// or absolute paths → convert via Capacitor.convertFileSrc
* iOS returns raw base64 (no data: prefix) → prepend the data URI scheme
*/
function normalizeScannedImage(raw) {
if (!raw) return null
if (raw.startsWith('data:')) return raw
if (raw.startsWith('http://') || raw.startsWith('https://') || raw.startsWith('content://')) return raw
if (raw.startsWith('/') || raw.startsWith('file://')) return Capacitor.convertFileSrc(raw)
// iOS base64 without prefix
return `data:image/jpeg;base64,${raw}`
}
/**
* Hook for native document scanning via @capgo/capacitor-document-scanner.
*
* On native: opens the OS document scanner (edge detection, perspective correction).
* On web: `scanDocument` returns null — callers should fall back to their own camera UI.
*/
export function useDocumentScanner() {
const isNativeScanner = Capacitor.isNativePlatform()
const scanDocument = async ({ maxDocuments = 1, quality = 90, letUserAdjustCrop = true } = {}) => {
if (!isNativeScanner) return { image: null, cancelled: false }
try {
const { DocumentScanner } = await import('@capgo/capacitor-document-scanner')
const { scannedImages } = await DocumentScanner.scanDocument({
croppedImageQuality: quality,
maxNumDocuments: maxDocuments,
letUserAdjustCrop,
})
if (!scannedImages?.length) return { image: null, cancelled: true }
const normalized = normalizeScannedImage(scannedImages[0])
return { image: normalized, cancelled: false }
} catch (e) {
console.error('[DocumentScanner] scan failed:', e)
return { image: null, cancelled: false, error: e.message }
}
}
return { isNativeScanner, scanDocument }
}

View File

@@ -0,0 +1,72 @@
const ENABLED_KEY = 'ai_prompt_cache_enabled'
const ENTRY_PREFIX = 'ai_prompt_cache_'
const INDEX_KEY = 'ai_prompt_cache_index'
function djb2(str) {
let hash = 5381
for (let i = 0; i < str.length; i++) {
hash = ((hash << 5) + hash) ^ str.charCodeAt(i)
hash = hash >>> 0
}
return hash.toString(36)
}
export function isCacheEnabled() {
try {
return localStorage.getItem(ENABLED_KEY) === 'true'
} catch {
return false
}
}
export function setCacheEnabled(enabled) {
try {
localStorage.setItem(ENABLED_KEY, String(enabled))
} catch { /* ignore */ }
}
export function hashContent(content) {
return djb2(typeof content === 'string' ? content : JSON.stringify(content))
}
function getIndex() {
try {
return JSON.parse(localStorage.getItem(INDEX_KEY) || '[]')
} catch {
return []
}
}
export function getCached(hash) {
if (!isCacheEnabled()) return null
try {
const raw = localStorage.getItem(ENTRY_PREFIX + hash)
return raw ? JSON.parse(raw) : null
} catch {
return null
}
}
export function setCached(hash, value) {
if (!isCacheEnabled()) return
try {
localStorage.setItem(ENTRY_PREFIX + hash, JSON.stringify(value))
const index = getIndex()
if (!index.includes(hash)) {
index.push(hash)
localStorage.setItem(INDEX_KEY, JSON.stringify(index))
}
} catch { /* storage full, ignore */ }
}
export function getCacheStats() {
return { count: getIndex().length }
}
export function clearCache() {
const index = getIndex()
index.forEach(h => {
try { localStorage.removeItem(ENTRY_PREFIX + h) } catch { /* ignore */ }
})
try { localStorage.removeItem(INDEX_KEY) } catch { /* ignore */ }
}

View File

@@ -0,0 +1,141 @@
import { Capacitor } from '@capacitor/core'
import { getCached, hashContent, setCached } from './AIPromptCache'
// Native-only local AI service using @capacitor/local-llm.
// On web, all methods return 'unavailable' / null — no WebLLM.
class LocalAIService {
constructor() {
this._availability = null
this._sessionId = 'donetick-summary'
this._warmedUp = false
}
get isNative() {
return Capacitor.isNativePlatform()
}
async checkAvailability() {
if (!this.isNative) {
this._availability = 'unavailable'
return 'unavailable'
}
try {
const { LocalLLM } = await import('@capacitor/local-llm')
const { status } = await LocalLLM.systemAvailability()
this._availability = status
return status
} catch (e) {
this._availability = 'unavailable'
return 'unavailable'
}
}
async getStatus() {
if (this._availability !== null) return this._availability
return this.checkAvailability()
}
async isAvailable() {
return (await this.getStatus()) === 'available'
}
resetAvailability() {
this._availability = null
}
async download(onStatusChange) {
if (!this.isNative) return
try {
const { LocalLLM } = await import('@capacitor/local-llm')
if (onStatusChange) {
LocalLLM.addListener('systemAvailabilityChange', ({ status }) => {
this._availability = status
onStatusChange(status)
})
}
await LocalLLM.download()
} catch {
// download not available on iOS, ignore
}
}
async warmup() {
if (this._warmedUp || !this.isNative) return
try {
const { LocalLLM } = await import('@capacitor/local-llm')
await LocalLLM.warmup({ sessionId: this._sessionId })
this._warmedUp = true
} catch {
// non-fatal
}
}
async _nativePrompt(text) {
await this.warmup()
try {
const { LocalLLM } = await import('@capacitor/local-llm')
const { text: out } = await LocalLLM.prompt({ prompt: text, sessionId: this._sessionId })
return out?.trim() || null
} finally {
try {
const { LocalLLM } = await import('@capacitor/local-llm')
await LocalLLM.endSession({ sessionId: this._sessionId })
this._warmedUp = false
} catch { /* ignore */ }
}
}
// Plain chat — no tools. Returns answer string or null.
async plainChat(messages) {
const available = await this.isAvailable()
if (!available) return null
const cacheHash = hashContent(['plain', ...messages])
const cached = getCached(cacheHash)
if (cached) return cached
if (!this.isNative) return null
try {
const systemMsg = messages.find(m => m.role === 'system')?.content || ''
const userMsg = messages.find(m => m.role === 'user')?.content || ''
const result = await this._nativePrompt(`${systemMsg}\n\nUser: ${userMsg}\nAssistant:`)
if (result) setCached(cacheHash, result)
return result
} catch (e) {
console.error('[LocalAI] plainChat() failed:', e)
return null
}
}
// Returns the summary string or null if LLM is unavailable
async summarize(prompt) {
const available = await this.isAvailable()
if (!available) return null
const cacheHash = hashContent(prompt)
const cached = getCached(cacheHash)
if (cached) return cached
if (!this.isNative) return null
try {
await this.warmup()
const { LocalLLM } = await import('@capacitor/local-llm')
const { text } = await LocalLLM.prompt({ prompt, sessionId: this._sessionId })
const result = text?.trim() || null
if (result) setCached(cacheHash, result)
return result
} catch {
return null
} finally {
try {
const { LocalLLM } = await import('@capacitor/local-llm')
await LocalLLM.endSession({ sessionId: this._sessionId })
this._warmedUp = false
} catch { /* ignore */ }
}
}
}
export const localAIService = new LocalAIService()

View File

@@ -1,4 +1,4 @@
import { Add } from '@mui/icons-material'
import { Add, CameraAlt } from '@mui/icons-material'
import { Box, Button, Typography } from '@mui/joy'
import { useMediaQuery } from '@mui/material'
import * as chrono from 'chrono-node'
@@ -28,6 +28,7 @@ import DueDatePickerField from './DueDatePickerField'
import LabelsPickerField from './LabelsPickerField'
import LearnMoreButton from './LearnMore'
import NotificationPickerField from './NotificationPickerField'
import PhotoTaskModal from './PhotoTaskModal'
import PriorityPickerField from './PriorityPickerField'
import RepeatPickerField from './RepeatPickerField'
import RichTextEditor from './RichTextEditor'
@@ -103,6 +104,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
const [showKeyboardShortcuts, setShowKeyboardShortcuts] = useState(false)
const [projectId, setProjectId] = useState(getInitialProject())
const [attachments, setAttachments] = useState([])
const [photoModalOpen, setPhotoModalOpen] = useState(false)
// Priority colors
const priorityColors = {
@@ -543,6 +545,23 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
createChore()
}
const handleTaskExtracted = ({ taskName, description: extractedDesc, dueDate: extractedDue }) => {
if (taskName) {
processText(taskName)
}
if (extractedDesc) {
setDescription(extractedDesc)
setHasDescription(true)
}
if (extractedDue) {
const m = moment(new Date(extractedDue))
if (m.isValid()) {
setDueDateOnly(m.format('YYYY-MM-DD'))
setDueDate(m.endOf('day').format('YYYY-MM-DDTHH:mm:ss'))
}
}
}
const handleCloseModal = forceRefetch => {
onClose(forceRefetch)
setTaskText('')
@@ -664,6 +683,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
}
return (
<>
<ResponsiveModal
open={isModalOpen}
onClose={handleCloseModal}
@@ -766,6 +786,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
placeholder='Type your task...'
onChange={text => {
setTaskText(text)
if (!text) setTaskTitle('')
}}
customRenderer={renderedParts}
onEnterPressed={handleEnterPressed}
@@ -909,6 +930,14 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
Description
</Button>
)}
<Button
startDecorator={<CameraAlt />}
variant='plain'
size='sm'
onClick={() => setPhotoModalOpen(true)}
>
Scan Photo
</Button>
{!hasSubTasks && (
<Button
@@ -935,6 +964,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
<RichTextEditor
ref={richTextEditorRef}
onChange={setDescription}
value={description || ''}
entityType={'chore_description'}
/>
</div>
@@ -952,6 +982,12 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
</Box>
)}
</ResponsiveModal>
<PhotoTaskModal
open={photoModalOpen}
onClose={() => setPhotoModalOpen(false)}
onTaskExtracted={handleTaskExtracted}
/>
</>
)
}

View File

@@ -0,0 +1,595 @@
import {
ArrowBack,
CameraAlt,
CheckCircle,
Close,
DocumentScanner,
PhotoCamera,
Replay,
TextSnippet,
} from '@mui/icons-material'
import {
Box,
Button,
CircularProgress,
IconButton,
LinearProgress,
Typography,
} from '@mui/joy'
import { useCallback, useEffect, useRef, useState } from 'react'
import { useDocumentScanner } from '../../hooks/useDocumentScanner'
import { useResponsiveModal } from '../../hooks/useResponsiveModal'
import { localAIService } from '../../service/LocalAIService'
const SYSTEM_PROMPT = `You are helping create tasks for a household task management app.
Given OCR text extracted from a photo, identify the most useful task a person should add to their task list.
The task title should always start with an action verb when possible.
Examples:
Bill -> "Pay water bill"
Appointment -> "Attend eye doctor appointment"
Invitation -> "RSVP for wedding"
Renewal Notice -> "Renew vehicle registration"
Package Notice -> "Pick up package"
School Form -> "Complete school permission form"
Action Priority Rules:
1. Payments and bills
2. Deadlines and renewals
3. Appointments
4. Required forms
5. Informational actions (view, read, review)
Rules:
Generate at most one task.
Focus on the most important action.
Extract due dates and deadlines.
Use appointment dates as due dates when appropriate.
Do not invent information.
If the content contains no actionable item, return null values.
Include any important ID or URL or instructions in the description
Titles must be specific and useful at a glance.
Include the organization, provider, event, or subject when available.
Avoid generic document names.
Return valid JSON only.
Output:
{
"taskName": string | null,
"description": string | null,
"dueDate": string | null,
"confidence": number
}`
async function runNativeOCR(imageSource) {
const { Ocr } = await import('@jcesarmobile/capacitor-ocr')
// Convert Capacitor WebView file URL → native file:// URL the plugin can read
const image = imageSource.includes('/_capacitor_file_/')
? 'file://' + imageSource.replace(/^https?:\/\/localhost\/_capacitor_file_/, '')
: imageSource
const result = await Ocr.process({ image })
return result.results.map(r => r.text).join('\n').trim()
}
async function runOCR(imageSource, onProgress) {
const { createWorker } = await import('tesseract.js')
const worker = await createWorker('eng', 1, {
logger: m => {
if (m.status === 'recognizing text' && onProgress) {
onProgress(Math.round(m.progress * 100))
}
},
})
const { data } = await worker.recognize(imageSource)
await worker.terminate()
return data.text?.trim() || ''
}
async function extractTaskFromOCR(ocrText) {
const messages = [
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: `OCR Text:\n${ocrText}` },
]
const result = await localAIService.plainChat(messages)
if (!result) return null
const jsonMatch = result.match(/\{[\s\S]*\}/)
if (!jsonMatch) return null
try {
return JSON.parse(jsonMatch[0])
} catch {
return null
}
}
const PhotoTaskModal = ({ open, onClose, onTaskExtracted }) => {
const { ResponsiveModal } = useResponsiveModal()
const { isNativeScanner, scanDocument } = useDocumentScanner()
const videoRef = useRef(null)
const canvasRef = useRef(null)
const streamRef = useRef(null)
const fileInputRef = useRef(null)
const [phase, setPhase] = useState('capture') // capture | preview | ocr | llm | done | error
const [capturedImage, setCapturedImage] = useState(null)
const [ocrProgress, setOcrProgress] = useState(0)
const [ocrText, setOcrText] = useState('')
const [ocrMethod, setOcrMethod] = useState('tesseract')
const [showRawText, setShowRawText] = useState(false)
const [taskResult, setTaskResult] = useState(null)
const [errorMsg, setErrorMsg] = useState('')
const [cameraAvailable, setCameraAvailable] = useState(true)
const startCamera = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: 'environment' },
})
streamRef.current = stream
if (videoRef.current) {
videoRef.current.srcObject = stream
}
setCameraAvailable(true)
} catch {
setCameraAvailable(false)
}
}, [])
const stopCamera = useCallback(() => {
if (streamRef.current) {
streamRef.current.getTracks().forEach(t => t.stop())
streamRef.current = null
}
}, [])
useEffect(() => {
if (open && phase === 'capture' && !isNativeScanner) {
startCamera()
}
return () => {
stopCamera()
}
}, [open, phase, isNativeScanner, startCamera, stopCamera])
const handleCapture = () => {
if (!videoRef.current || !canvasRef.current) return
const video = videoRef.current
const canvas = canvasRef.current
canvas.width = video.videoWidth
canvas.height = video.videoHeight
canvas.getContext('2d').drawImage(video, 0, 0)
const dataUrl = canvas.toDataURL('image/jpeg', 0.9)
setCapturedImage(dataUrl)
stopCamera()
setPhase('preview')
}
const handleFileSelect = e => {
const file = e.target.files?.[0]
if (!file) return
const reader = new FileReader()
reader.onload = ev => {
setCapturedImage(ev.target.result)
stopCamera()
setPhase('preview')
}
reader.readAsDataURL(file)
}
const handleBackToPreview = () => {
setOcrText('')
setTaskResult(null)
setErrorMsg('')
setOcrProgress(0)
setShowRawText(false)
setPhase('preview')
}
const handleProcess = async (method = 'tesseract') => {
setOcrMethod(method)
setPhase('ocr')
setOcrProgress(0)
setErrorMsg('')
setShowRawText(false)
try {
let text
if (method === 'native') {
try {
text = await runNativeOCR(capturedImage)
} catch {
throw new Error('Native OCR is only available on iOS and Android devices.')
}
} else {
text = await runOCR(capturedImage, pct => setOcrProgress(pct))
}
setOcrText(text)
if (!text) {
setErrorMsg('No text found in the image. Please try a clearer photo.')
setPhase('error')
return
}
setPhase('llm')
const task = await extractTaskFromOCR(text)
if (!task || !task.taskName) {
setErrorMsg('Could not identify a task from this image. Please try a different photo.')
setPhase('error')
return
}
setTaskResult(task)
setPhase('done')
} catch (e) {
setErrorMsg(`Processing failed: ${e.message || 'Unknown error'}`)
setPhase('error')
}
}
const handleRetake = () => {
setCapturedImage(null)
setOcrText('')
setTaskResult(null)
setErrorMsg('')
setOcrProgress(0)
setShowRawText(false)
setPhase('capture')
}
const handleNativeScan = async () => {
const { image, cancelled, error } = await scanDocument()
if (cancelled) return
if (error || !image) {
setErrorMsg(error ? `Scanner error: ${error}` : 'Scan cancelled or failed.')
setPhase('error')
return
}
setCapturedImage(image)
stopCamera()
setPhase('preview')
}
const handleConfirm = () => {
if (taskResult) {
onTaskExtracted(taskResult)
}
handleClose()
}
const handleClose = () => {
stopCamera()
setCapturedImage(null)
setOcrText('')
setTaskResult(null)
setErrorMsg('')
setOcrProgress(0)
setShowRawText(false)
setPhase('capture')
onClose()
}
const isProcessing = phase === 'ocr' || phase === 'llm'
return (
<ResponsiveModal
open={open}
onClose={handleClose}
size='md'
fullWidth
title='Scan photo to create task'
>
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
{(phase === 'capture' || phase === 'preview') && (
<Box
sx={{
position: 'relative',
width: '100%',
borderRadius: 'md',
overflow: 'hidden',
bgcolor: 'background.level1',
minHeight: 240,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
}}
>
{phase === 'capture' && !isNativeScanner && cameraAvailable && (
<video
ref={videoRef}
autoPlay
playsInline
muted
style={{ width: '100%', display: 'block' }}
/>
)}
{phase === 'capture' && isNativeScanner && (
<Box sx={{ textAlign: 'center', p: 4 }}>
<DocumentScanner sx={{ fontSize: 64, opacity: 0.4, mb: 1 }} />
<Typography level='body-sm' sx={{ opacity: 0.6 }}>
Tap &quot;Scan Document&quot; to open the scanner
</Typography>
</Box>
)}
{phase === 'capture' && !isNativeScanner && !cameraAvailable && (
<Box sx={{ textAlign: 'center', p: 3 }}>
<CameraAlt sx={{ fontSize: 48, opacity: 0.5, mb: 1 }} />
<Typography level='body-sm' sx={{ opacity: 0.7 }}>
Camera not available
</Typography>
</Box>
)}
{phase === 'preview' && capturedImage && (
<img
src={capturedImage}
alt='Captured document'
style={{ width: '100%', display: 'block' }}
/>
)}
</Box>
)}
<canvas ref={canvasRef} style={{ display: 'none' }} />
{isProcessing && (
<Box
sx={{
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
gap: 2,
py: 4,
}}
>
{capturedImage && (
<img
src={capturedImage}
alt='Processing'
style={{
width: '100%',
borderRadius: 8,
opacity: 0.6,
maxHeight: 200,
objectFit: 'contain',
}}
/>
)}
<CircularProgress size='md' />
{phase === 'ocr' && ocrMethod === 'tesseract' && (
<>
<Typography level='body-sm'>
Reading text from image {ocrProgress}%
</Typography>
<LinearProgress determinate value={ocrProgress} sx={{ width: '100%' }} />
</>
)}
{phase === 'ocr' && ocrMethod === 'native' && (
<Typography level='body-sm'>Running native OCR</Typography>
)}
{phase === 'llm' && (
<Typography level='body-sm'>Identifying task with AI</Typography>
)}
</Box>
)}
{phase === 'done' && taskResult && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1.5 }}>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
<CheckCircle color='success' />
<Typography level='title-sm'>Task identified</Typography>
</Box>
<Box
sx={{
p: 1.5,
borderRadius: 'md',
bgcolor: 'background.level1',
border: '1px solid',
borderColor: 'divider',
}}
>
<Typography level='title-sm'>{taskResult.taskName}</Typography>
{taskResult.description && (
<Typography level='body-xs' sx={{ mt: 0.5, opacity: 0.8 }}>
{taskResult.description}
</Typography>
)}
{taskResult.dueDate && (
<Typography level='body-xs' sx={{ mt: 0.5, opacity: 0.7 }}>
Due: {taskResult.dueDate}
</Typography>
)}
</Box>
{ocrText && (
<>
<Button
size='sm'
variant='plain'
color='neutral'
startDecorator={<TextSnippet />}
onClick={() => setShowRawText(v => !v)}
sx={{ alignSelf: 'flex-start' }}
>
{showRawText ? 'Hide Raw Text' : 'Show Raw Text'}
</Button>
{showRawText && (
<Box
sx={{
p: 1.5,
borderRadius: 'md',
bgcolor: 'background.level2',
border: '1px solid',
borderColor: 'divider',
maxHeight: 180,
overflowY: 'auto',
}}
>
<Typography
level='body-xs'
sx={{ whiteSpace: 'pre-wrap', fontFamily: 'monospace' }}
>
{ocrText}
</Typography>
</Box>
)}
</>
)}
</Box>
)}
{phase === 'error' && (
<Box
sx={{
p: 2,
borderRadius: 'md',
bgcolor: 'danger.softBg',
color: 'danger.softColor',
}}
>
<Typography level='body-sm'>{errorMsg}</Typography>
</Box>
)}
<Box sx={{ display: 'flex', gap: 1, justifyContent: 'flex-end' }}>
{phase === 'capture' && (
<>
<Button
variant='outlined'
color='neutral'
startDecorator={<PhotoCamera />}
onClick={() => fileInputRef.current?.click()}
>
Upload Photo
</Button>
<input
ref={fileInputRef}
type='file'
accept='image/*'
style={{ display: 'none' }}
onChange={handleFileSelect}
/>
{isNativeScanner ? (
<Button
variant='solid'
color='primary'
startDecorator={<DocumentScanner />}
onClick={handleNativeScan}
>
Scan Document
</Button>
) : (
cameraAvailable && (
<Button
variant='solid'
color='primary'
startDecorator={<CameraAlt />}
onClick={handleCapture}
>
Capture
</Button>
)
)}
</>
)}
{phase === 'preview' && (
<>
<Button
variant='outlined'
color='neutral'
startDecorator={<Replay />}
onClick={handleRetake}
>
Retake
</Button>
{isNativeScanner && (
<Button
variant='outlined'
color='primary'
onClick={() => handleProcess('native')}
>
Process Natively
</Button>
)}
<Button
variant='solid'
color='primary'
onClick={() => handleProcess('tesseract')}
>
Process Image
</Button>
</>
)}
{phase === 'error' && (
<>
<Button
variant='outlined'
color='neutral'
startDecorator={<ArrowBack />}
onClick={handleBackToPreview}
>
Back
</Button>
<Button
variant='outlined'
color='neutral'
startDecorator={<Replay />}
onClick={handleRetake}
>
Retake
</Button>
</>
)}
{phase === 'done' && (
<>
<Button
variant='outlined'
color='neutral'
startDecorator={<ArrowBack />}
onClick={handleBackToPreview}
>
Back
</Button>
<Button
variant='outlined'
color='neutral'
startDecorator={<Replay />}
onClick={handleRetake}
>
Retake
</Button>
<Button variant='solid' color='primary' onClick={handleConfirm}>
Create Task
</Button>
</>
)}
{!isProcessing && (
<IconButton
variant='plain'
color='neutral'
onClick={handleClose}
sx={{ ml: 'auto' }}
>
<Close />
</IconButton>
)}
</Box>
</Box>
</ResponsiveModal>
)
}
export default PhotoTaskModal