jonas/donetick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(voice): harden native recognition loop and match labels to existing ones only

Browse Source 
Add heartbeat/timeout recovery for silently-dead Android recognizer sessions,
wait for and prefer the late-arriving final transcript over the interim
partial, and guard against duplicate commits when it lands after restart.
Bias native recognition toward circle member/label names via contextualStrings
so unfamiliar names aren't auto-corrected. Restrict spoken "label X" to
existing labels (exact or closest fuzzy match) instead of creating new ones,
removing the now-unused label-creation flow from AddTaskModal/VoicePanel/
parseVoiceTask.
This commit is contained in:
Mo Tarbin
2026-07-26 21:41:56 -04:00
parent a133b271d0
commit 09c3dbbebc
7 changed files with 850 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/views/components/VoiceToTask/VoicePanel.jsx
View File

@@ -144,14 +144,6 @@ const buildChips = (effective, frequencyLabel, { members, currentUserId }) => {
label: name,
})
})
effective.newLabels.forEach(label => {
chips.push({
key: `new-label-${label.name}`,
color: 'warning',
icon: <Sell sx={{ fontSize: 12 }} />,
label: `${label.name} · new`,
})
})
if (effective.isAnyone) {
chips.push({
key: 'assignee',
@@ -436,9 +428,10 @@ const VoicePanel = ({
patchSegment,
reset,
isNative,
} = useVoiceToTask({ members })
} = useVoiceToTask({ members, userLabels })
const [creating, setCreating] = useState(false)
const autoStartedRef = useRef(false)
const segmentsScrollRef = useRef(null)
const parseCtx = useMemo(
() => ({ userLabels, members, currentUserId: userProfile?.id }),
@@ -459,6 +452,12 @@ const VoicePanel = ({
}
}, [open, startHandsFree])
// Keep the newest captured task visible as more are added
useEffect(() => {
const el = segmentsScrollRef.current
if (el) el.scrollTop = el.scrollHeight
}, [segments.length])
if (!open) return null
const isListening = phase === 'listening'
@@ -552,6 +551,7 @@ const VoicePanel = ({
{/* ── Captured task cards ── */}
{segments.length > 0 && (
<Box
ref={segmentsScrollRef}
sx={{
px: 1.5,
pt: 1.25,

1
src/views/components/VoiceToTask/parseVoiceTask.js
View File

@@ -124,7 +124,6 @@ export const parseVoiceTask = (
points: points.result ?? null,
labelIds,
labelNames: (labels.result || []).map(label => label.name),
newLabels: labels.newLabels || [],
assignees,
isAnyone,
frequency: repeat.result,

110
src/views/components/VoiceToTask/useVoiceToTask.js
View File

@@ -32,8 +32,19 @@ const haptic = async kind => {
}
}
// Vocabulary fed to the native recognizer as a biasing hint so unfamiliar
// names/labels aren't auto-corrected to a dictionary word (e.g. "Moutaz" →
// "Models"). Best-effort only — unsupported on iOS <13-without-on-device and
// Android <13, which is why the normalizer also does fuzzy post-matching.
const buildVocabulary = (members, userLabels) => [
...members.flatMap(m =>
[m.displayName, m.displayName?.split(/\s+/)[0], m.username].filter(Boolean),
),
...userLabels.map(l => l.name).filter(Boolean),
]
// phases: idle | listening | review | denied
export function useVoiceToTask({ members = [] } = {}) {
export function useVoiceToTask({ members = [], userLabels = [] } = {}) {
const [phase, setPhase] = useState('idle')
const [isLocked, setIsLocked] = useState(false)
const [partialText, setPartialText] = useState('')
@@ -44,6 +55,10 @@ export function useVoiceToTask({ members = [] } = {}) {
const segmentsRef = useRef([])
const membersRef = useRef(members)
membersRef.current = members
const userLabelsRef = useRef(userLabels)
userLabelsRef.current = userLabels
const vocabularyRef = useRef(buildVocabulary(members, userLabels))
vocabularyRef.current = buildVocabulary(members, userLabels)
const phaseRef = useRef(phase)
phaseRef.current = phase
@@ -54,6 +69,12 @@ export function useVoiceToTask({ members = [] } = {}) {
const pressStartedListeningRef = useRef(false)
const lastActivityRef = useRef(0)
const watchdogRef = useRef(null)
// While the mic is held (not locked), a mid-hold restart (Android session
// limits, forced silence boundary) shouldn't split into a new task — the
// user is still holding the button, so it's still one entry. This tracks
// which segment is the "active" one for the current hold to merge onto;
// reset to null on release so the *next* hold starts a fresh entry.
const activeHoldSegmentIdRef = useRef(null)
const applySegments = useCallback(next => {
segmentsRef.current = next
@@ -64,6 +85,7 @@ export function useVoiceToTask({ members = [] } = {}) {
rawText => {
const normalized = normalizeSpokenText(rawText, {
members: membersRef.current,
userLabels: userLabelsRef.current,
})
const { text, dropPrevious } = applyScratchThat(normalized)
const pieces = splitSpokenSegments(text)
@@ -71,14 +93,52 @@ export function useVoiceToTask({ members = [] } = {}) {
let base = segmentsRef.current
if (dropPrevious && base.length > 0) {
const dropped = base[base.length - 1]
base = base.slice(0, -1)
if (activeHoldSegmentIdRef.current === dropped.id) {
activeHoldSegmentIdRef.current = null
}
haptic('medium')
}
if (pieces.length > 0) haptic('light')
applySegments([
...base,
...pieces.map(piece => ({ id: generateUUID(), text: piece })),
])
if (pieces.length === 0) {
applySegments(base)
return
}
haptic('light')
if (!lockedRef.current) {
// Hold-to-talk: the first piece continues the entry already active
// for this hold (if any); only a spoken separator within the same
// commit starts additional new entries.
const activeIndex = base.findIndex(
s => s.id === activeHoldSegmentIdRef.current,
)
if (activeIndex !== -1) {
const merged = [...base]
merged[activeIndex] = {
...merged[activeIndex],
text: `${merged[activeIndex].text} ${pieces[0]}`.trim(),
}
const rest = pieces.slice(1).map(piece => ({
id: generateUUID(),
text: piece,
}))
if (rest.length > 0) {
activeHoldSegmentIdRef.current = rest[rest.length - 1].id
}
applySegments([...merged, ...rest])
return
}
}
const newPieces = pieces.map(piece => ({
id: generateUUID(),
text: piece,
}))
if (!lockedRef.current) {
activeHoldSegmentIdRef.current = newPieces[newPieces.length - 1].id
}
applySegments([...base, ...newPieces])
},
[applySegments],
)
@@ -91,6 +151,9 @@ export function useVoiceToTask({ members = [] } = {}) {
await voiceInputService.stop()
setPartialText('')
setIsLocked(false)
// Release ends the current hold — the next hold-press starts a fresh
// entry rather than continuing to merge onto this one
activeHoldSegmentIdRef.current = null
// stop() commits any buffered partial synchronously through onSegment,
// so the ref is up to date by the time we read it
setPhase(segmentsRef.current.length > 0 ? 'review' : 'idle')
@@ -104,19 +167,25 @@ export function useVoiceToTask({ members = [] } = {}) {
return false
}
lastActivityRef.current = Date.now()
await voiceInputService.start({
onPartial: text => {
lastActivityRef.current = Date.now()
setPartialText(
normalizeSpokenText(text, { members: membersRef.current }),
)
await voiceInputService.start(
{
onPartial: text => {
lastActivityRef.current = Date.now()
setPartialText(
normalizeSpokenText(text, {
members: membersRef.current,
userLabels: userLabelsRef.current,
}),
)
},
onSegment: commitSegment,
onError: () => {
setPhase('denied')
},
onStateChange: () => {},
},
onSegment: commitSegment,
onError: () => {
setPhase('denied')
},
onStateChange: () => {},
})
vocabularyRef.current,
)
setPhase('listening')
haptic('medium')
@@ -156,7 +225,9 @@ export function useVoiceToTask({ members = [] } = {}) {
const held = Date.now() - pressStartedAtRef.current
if (pressStartedListeningRef.current) {
if (held < TAP_THRESHOLD_MS) {
// Quick tap → hands-free lock
// Quick tap → hands-free lock; from here on, silence boundaries
// should start new entries again, not merge onto the last one
activeHoldSegmentIdRef.current = null
setIsLocked(true)
} else {
// Hold-to-talk → release ends the capture
@@ -211,6 +282,7 @@ export function useVoiceToTask({ members = [] } = {}) {
applySegments([])
setPartialText('')
setIsLocked(false)
activeHoldSegmentIdRef.current = null
setPhase('idle')
}, [applySegments])

107
src/views/components/VoiceToTask/voiceNormalizer.js
View File

@@ -1,7 +1,7 @@
// Deterministic transforms that turn spoken language into the typed syntax
// CustomParsers understands. No LLM — instant, predictable, fully offline.
//
// "label groceries" → "#groceries"
// "label groceries" → "#groceries" (only when it matches an existing label)
// "assign to Sarah" → "@Sarah" (only when Sarah is a circle member)
// "worth five points" → "*5"
// "p one" / "top priority" → "priority 1" (parsePriority already handles that)
@@ -55,22 +55,23 @@ const normalizePoints = text =>
(_, n) => `*${NUMBER_WORDS[n.toLowerCase()] || n} points`,
)
const normalizeLabels = text =>
text.replace(
/\b(?:with\s+)?(?:hash\s?tag|labell?ed(?:\s+as)?|label|tagged(?:\s+as)?|tag)\s+([\p{L}\p{N}_]+)/giu,
'#$1',
)
// "assign to Sarah" / "assigned to Sarah" / "assign Sarah" / "for Sarah".
// Speech engines spell names their own way ("Sara" for Sarah) and add
// punctuation, so exact display-name matching alone misses real speech —
// an edit-distance-1 fuzzy pass catches those, but only after an explicit
// assign verb so ordinary words never convert.
// Speech engines spell names their own way — and worse, can auto-correct an
// unfamiliar name to an unrelated dictionary word entirely ("Moutaz" heard as
// "Models"), which plain edit-distance can't recover (too many edits apart).
// But right after an assign verb, the next word has essentially no other
// legitimate reading — it IS a name — so we take the *relative best* match
// among circle members rather than requiring it to be objectively close.
// A same-first-letter guard keeps this from firing on totally unrelated
// words. Contextual-string biasing in VoiceInputService is the primary
// defense (it can make the recognizer hear "Moutaz" correctly in the first
// place); this is the fallback for when biasing isn't supported or still
// mishears.
const ASSIGN_VERB = '(?:assign(?:ed|ee)?(?:\\s+(?:this|it))?(?:\\s+to)?|for)'
const STRICT_ASSIGN_VERB = '(?:assign(?:ed|ee)?(?:\\s+(?:this|it))?(?:\\s+to)?)'
const MIN_MATCH_SCORE = 0.2
const levenshtein = (a, b) => {
if (Math.abs(a.length - b.length) > 1) return 2
const prev = Array.from({ length: b.length + 1 }, (_, i) => i)
for (let i = 1; i <= a.length; i++) {
let diag = prev[0]
@@ -88,6 +89,9 @@ const levenshtein = (a, b) => {
return prev[b.length]
}
const similarity = (a, b) =>
1 - levenshtein(a, b) / Math.max(a.length, b.length)
const memberNameVariants = member =>
[
member.displayName,
@@ -95,19 +99,75 @@ const memberNameVariants = member =>
member.username,
].filter(n => n && n.length > 1)
const findMemberFuzzy = (candidate, members) => {
// Best-scoring item for `candidate` among `items`, requiring only that it
// beats all others and shares a first letter — not an absolute closeness
// threshold. `getVariants` returns the name strings to compare a given item
// against (e.g. a member's display name/first name/username, or a label's
// name). Shared by assignee and label matching since both face the same
// problem: ASR is least confident on exactly the words that matter here.
const findBestFuzzyMatch = (candidate, items, getVariants) => {
const c = candidate.toLowerCase()
let close = null
for (const member of members) {
for (const name of memberNameVariants(member)) {
if (c.length < 3) return null
let best = null
let bestScore = MIN_MATCH_SCORE
for (const item of items) {
for (const name of getVariants(item)) {
const n = name.toLowerCase()
if (n === c) return member
if (!close && n.length >= 4 && c.length >= 4 && levenshtein(n, c) <= 1) {
close = member
if (n === c) return item
if (n.length < 3 || n[0] !== c[0]) continue
const score = similarity(n, c)
if (score > bestScore) {
bestScore = score
best = item
}
}
}
return close
return best
}
const findMemberFuzzy = (candidate, members) =>
findBestFuzzyMatch(candidate, members, memberNameVariants)
// "label groceries" / "tag groceries" / "labeled as groceries" — only ever
// converts to a label that already exists (matched exactly or as the closest
// existing one), never invents a new one. Restricted to single-word label
// names: CustomParsers' hashtag pattern (#([\p{L}\p{N}_]+)) can't span a
// space, so a multi-word label like "Home Maintenance" could never be
// represented as "#Home Maintenance" anyway — same limitation typing it by
// hand would hit.
const LABEL_VERB =
'(?:with\\s+)?(?:hash\\s?tag|labell?ed(?:\\s+as)?|label|tagged(?:\\s+as)?|tag)'
const normalizeLabels = (text, userLabels = []) => {
const singleWordLabels = userLabels.filter(l => l.name && !/\s/.test(l.name))
let out = text
// Exact pass first so a clean spoken match always wins over the fuzzy pass
const byLengthDesc = [...singleWordLabels].sort(
(a, b) => b.name.length - a.name.length,
)
for (const label of byLengthDesc) {
out = out.replace(
new RegExp(
`\\b${LABEL_VERB}\\s+${escapeRegex(label.name)}\\b[,.]?`,
'gi',
),
`#${label.name}`,
)
}
// Fuzzy pass — the spoken word after the verb, matched against the closest
// existing single-word label
out = out.replace(
new RegExp(`\\b${LABEL_VERB}\\s+([\\p{L}][\\p{L}'-]*)[,.]?`, 'giu'),
(match, candidate) => {
const label = findBestFuzzyMatch(candidate, singleWordLabels, l => [
l.name,
])
return label ? `#${label.name}` : match
},
)
return out
}
const normalizeAssignees = (text, members = []) => {
@@ -146,11 +206,14 @@ const normalizeAssignees = (text, members = []) => {
return out
}
export const normalizeSpokenText = (text, { members = [] } = {}) => {
export const normalizeSpokenText = (
text,
{ members = [], userLabels = [] } = {},
) => {
let out = stripFillers(text)
out = normalizePriority(out)
out = normalizePoints(out)
out = normalizeLabels(out)
out = normalizeLabels(out, userLabels)
out = normalizeAssignees(out, members)
return out.replace(/\s+/g, ' ').trim()
}