Add Initial Voice Input

This commit is contained in:
Mo Tarbin
2026-07-20 01:41:42 -04:00
parent 65c774d0d0
commit e0257445ef
17 changed files with 1747 additions and 17 deletions

View File

@@ -9,10 +9,12 @@ android {
apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
dependencies {
implementation project(':capacitor-community-speech-recognition')
implementation project(':capacitor-community-sqlite')
implementation project(':capacitor-app')
implementation project(':capacitor-browser')
implementation project(':capacitor-device')
implementation project(':capacitor-haptics')
implementation project(':capacitor-local-llm')
implementation project(':capacitor-local-notifications')
implementation project(':capacitor-network')

View File

@@ -117,4 +117,13 @@
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.NFC" />
<!-- Voice task capture -->
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<!-- Android 11+ package visibility for the system speech recognition service -->
<queries>
<intent>
<action android:name="android.speech.RecognitionService" />
</intent>
</queries>
</manifest>

View File

@@ -2,6 +2,9 @@
include ':capacitor-android'
project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor')
include ':capacitor-community-speech-recognition'
project(':capacitor-community-speech-recognition').projectDir = new File('../node_modules/@capacitor-community/speech-recognition/android')
include ':capacitor-community-sqlite'
project(':capacitor-community-sqlite').projectDir = new File('../node_modules/@capacitor-community/sqlite/android')
@@ -14,6 +17,9 @@ project(':capacitor-browser').projectDir = new File('../node_modules/@capacitor/
include ':capacitor-device'
project(':capacitor-device').projectDir = new File('../node_modules/@capacitor/device/android')
include ':capacitor-haptics'
project(':capacitor-haptics').projectDir = new File('../node_modules/@capacitor/haptics/android')
include ':capacitor-local-llm'
project(':capacitor-local-llm').projectDir = new File('../node_modules/@capacitor/local-llm/android')

View File

@@ -28,6 +28,10 @@
<string>This app needs access to camera to take photos to attach to task or use as profile photo</string>
<key>NSPhotoLibraryUsageDescription</key>
<string>This app needs access to photo library to select images to attach to task or use as profile photo</string>
<key>NSMicrophoneUsageDescription</key>
<string>This app uses the microphone to let you create tasks by speaking</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>This app uses on-device speech recognition to turn your voice into tasks</string>
<key>UIBackgroundModes</key>
<array/>
<key>UILaunchStoryboardName</key>

View File

@@ -11,10 +11,12 @@ install! 'cocoapods', :disable_input_output_paths => true
def capacitor_pods
pod 'Capacitor', :path => '../../node_modules/@capacitor/ios'
pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios'
pod 'CapacitorCommunitySpeechRecognition', :path => '../../node_modules/@capacitor-community/speech-recognition'
pod 'CapacitorCommunitySqlite', :path => '../../node_modules/@capacitor-community/sqlite'
pod 'CapacitorApp', :path => '../../node_modules/@capacitor/app'
pod 'CapacitorBrowser', :path => '../../node_modules/@capacitor/browser'
pod 'CapacitorDevice', :path => '../../node_modules/@capacitor/device'
pod 'CapacitorHaptics', :path => '../../node_modules/@capacitor/haptics'
pod 'CapacitorLocalLlm', :path => '../../node_modules/@capacitor/local-llm'
pod 'CapacitorLocalNotifications', :path => '../../node_modules/@capacitor/local-notifications'
pod 'CapacitorNetwork', :path => '../../node_modules/@capacitor/network'

View File

@@ -18,6 +18,8 @@ PODS:
- Capacitor
- CapacitorBrowser (8.0.3):
- Capacitor
- CapacitorCommunitySpeechRecognition (7.0.1):
- Capacitor
- CapacitorCommunitySqlite (8.1.0):
- Capacitor
- SQLCipher
@@ -25,6 +27,8 @@ PODS:
- CapacitorCordova (8.4.1)
- CapacitorDevice (8.0.2):
- Capacitor
- CapacitorHaptics (8.0.2):
- Capacitor
- CapacitorLocalLlm (1.0.0):
- Capacitor
- CapacitorLocalNotifications (8.2.0):
@@ -149,9 +153,11 @@ DEPENDENCIES:
- "Capacitor (from `../../node_modules/@capacitor/ios`)"
- "CapacitorApp (from `../../node_modules/@capacitor/app`)"
- "CapacitorBrowser (from `../../node_modules/@capacitor/browser`)"
- "CapacitorCommunitySpeechRecognition (from `../../node_modules/@capacitor-community/speech-recognition`)"
- "CapacitorCommunitySqlite (from `../../node_modules/@capacitor-community/sqlite`)"
- "CapacitorCordova (from `../../node_modules/@capacitor/ios`)"
- "CapacitorDevice (from `../../node_modules/@capacitor/device`)"
- "CapacitorHaptics (from `../../node_modules/@capacitor/haptics`)"
- "CapacitorLocalLlm (from `../../node_modules/@capacitor/local-llm`)"
- "CapacitorLocalNotifications (from `../../node_modules/@capacitor/local-notifications`)"
- "CapacitorNetwork (from `../../node_modules/@capacitor/network`)"
@@ -203,12 +209,16 @@ EXTERNAL SOURCES:
:path: "../../node_modules/@capacitor/app"
CapacitorBrowser:
:path: "../../node_modules/@capacitor/browser"
CapacitorCommunitySpeechRecognition:
:path: "../../node_modules/@capacitor-community/speech-recognition"
CapacitorCommunitySqlite:
:path: "../../node_modules/@capacitor-community/sqlite"
CapacitorCordova:
:path: "../../node_modules/@capacitor/ios"
CapacitorDevice:
:path: "../../node_modules/@capacitor/device"
CapacitorHaptics:
:path: "../../node_modules/@capacitor/haptics"
CapacitorLocalLlm:
:path: "../../node_modules/@capacitor/local-llm"
CapacitorLocalNotifications:
@@ -243,9 +253,11 @@ SPEC CHECKSUMS:
Capacitor: 35242afe195b1e53c58ca1b827d1b444c5e6602b
CapacitorApp: 449ffe26375e96f8aaaee625ac6e01e5c57c8650
CapacitorBrowser: c987c73d09d8bd3b5ec13f06338b1e14d5d2be69
CapacitorCommunitySpeechRecognition: 3e03566c44c2bb3b52391a33d4518b1adbdeb38f
CapacitorCommunitySqlite: eac6acfb852f46e7988fc59604d7f900498d354e
CapacitorCordova: eebe6bcf807b1b06f3f48237650f96bbcd0eef09
CapacitorDevice: 14cba6f88d1c3074cbf825fea977c8c526453ff8
CapacitorHaptics: 296f771ecd89c7a1bd92a7b6826a7d268e2e70f5
CapacitorLocalLlm: a05516151a02923a9e7dae9949d3817e85e321f0
CapacitorLocalNotifications: 2615aa008f608b95d3921a778ee1988abf1e6148
CapacitorNetwork: 8812ce60d11fb63d8f2e4ba51a49b2e59892ebe2
@@ -283,6 +295,6 @@ SPEC CHECKSUMS:
SQLCipher: eb79c64049cb002b4e9fcb30edb7979bf4706dfc
ZIPFoundation: dfd3d681c4053ff7e2f7350bc4e53b5dba3f5351
PODFILE CHECKSUM: 21b805bdbbb6ac4b8a3527ee8ef346ba0a55ef65
PODFILE CHECKSUM: 1099083fe561f8852fcef1bfcff4051f45db9770
COCOAPODS: 1.16.2

179
package-lock.json generated
View File

@@ -1,19 +1,22 @@
{
"name": "donetick",
"version": "1.2.15",
"version": "1.2.16",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "donetick",
"version": "1.2.15",
"version": "1.2.16",
"hasInstallScript": true,
"dependencies": {
"@capacitor-community/speech-recognition": "^7.0.1",
"@capacitor-community/sqlite": "^8.0.0",
"@capacitor/android": "^8.0.0",
"@capacitor/app": "^8.0.0",
"@capacitor/browser": "^8.0.0",
"@capacitor/core": "^8.0.0",
"@capacitor/device": "^8.0.0",
"@capacitor/haptics": "^8.0.2",
"@capacitor/ios": "^8.0.0",
"@capacitor/local-llm": "^1.0.0",
"@capacitor/local-notifications": "^8.0.0",
@@ -94,6 +97,7 @@
"eslint-plugin-sort-keys-fix": "^1.1.2",
"eslint-plugin-tailwindcss": "^3.13.1",
"husky": "^8.0.3",
"patch-package": "^8.0.1",
"postcss": "^8.4.32",
"prettier": "^3.1.1",
"prettier-plugin-tailwindcss": "^0.5.10",
@@ -1464,6 +1468,15 @@
"devOptional": true,
"license": "MIT"
},
"node_modules/@capacitor-community/speech-recognition": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/@capacitor-community/speech-recognition/-/speech-recognition-7.0.1.tgz",
"integrity": "sha512-ykpBZziR575X0eURO5vXaD9gVrXXC/7Ra2qql/2KP6/jxWOqAFuw4eKjSPjwdAgGE6a/Z+v8FJm4SVh57MVwDA==",
"license": "MIT",
"peerDependencies": {
"@capacitor/core": ">=7.0.0"
}
},
"node_modules/@capacitor-community/sqlite": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/@capacitor-community/sqlite/-/sqlite-8.1.0.tgz",
@@ -2057,6 +2070,15 @@
"@capacitor/core": ">=8.0.0"
}
},
"node_modules/@capacitor/haptics": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/@capacitor/haptics/-/haptics-8.0.2.tgz",
"integrity": "sha512-c2hZzRR5Fk1tbTvhG1jhh2XBAf3EhnIerMIb2sl7Mt41Gxx1fhBJFDa0/BI1IbY4loVepyyuqNC9820/GZuoWQ==",
"license": "MIT",
"peerDependencies": {
"@capacitor/core": ">=8.0.0"
}
},
"node_modules/@capacitor/ios": {
"version": "8.4.1",
"resolved": "https://registry.npmjs.org/@capacitor/ios/-/ios-8.4.1.tgz",
@@ -5518,6 +5540,13 @@
"node": ">=10.0.0"
}
},
"node_modules/@yarnpkg/lockfile": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz",
"integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==",
"dev": true,
"license": "BSD-2-Clause"
},
"node_modules/acorn": {
"version": "8.15.0",
"license": "MIT",
@@ -6767,6 +6796,22 @@
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
}
},
"node_modules/ci-info": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz",
"integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==",
"dev": true,
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/sibiraj-s"
}
],
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/classlist-polyfill": {
"version": "1.2.0",
"license": "Unlicense"
@@ -8677,6 +8722,16 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/find-yarn-workspace-root": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz",
"integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"micromatch": "^4.0.2"
}
},
"node_modules/flat-cache": {
"version": "3.2.0",
"dev": true,
@@ -10135,11 +10190,38 @@
"dev": true,
"license": "MIT"
},
"node_modules/json-stable-stringify": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz",
"integrity": "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==",
"dev": true,
"license": "MIT",
"dependencies": {
"call-bind": "^1.0.8",
"call-bound": "^1.0.4",
"isarray": "^2.0.5",
"jsonify": "^0.0.1",
"object-keys": "^1.1.1"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/json-stable-stringify-without-jsonify": {
"version": "1.0.1",
"dev": true,
"license": "MIT"
},
"node_modules/json-stable-stringify/node_modules/isarray": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
"integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==",
"dev": true,
"license": "MIT"
},
"node_modules/json-stringify-safe": {
"version": "5.0.1",
"dev": true,
@@ -10165,6 +10247,16 @@
"graceful-fs": "^4.1.6"
}
},
"node_modules/jsonify": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz",
"integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==",
"dev": true,
"license": "Public Domain",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/jsonparse": {
"version": "1.3.1",
"dev": true,
@@ -10235,6 +10327,16 @@
"node": ">=0.10.0"
}
},
"node_modules/klaw-sync": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz",
"integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.1.11"
}
},
"node_modules/kleur": {
"version": "4.1.5",
"dev": true,
@@ -11322,6 +11424,79 @@
"cross-spawn": "^7.0.3"
}
},
"node_modules/patch-package": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.1.tgz",
"integrity": "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@yarnpkg/lockfile": "^1.1.0",
"chalk": "^4.1.2",
"ci-info": "^3.7.0",
"cross-spawn": "^7.0.3",
"find-yarn-workspace-root": "^2.0.0",
"fs-extra": "^10.0.0",
"json-stable-stringify": "^1.0.2",
"klaw-sync": "^6.0.0",
"minimist": "^1.2.6",
"open": "^7.4.2",
"semver": "^7.5.3",
"slash": "^2.0.0",
"tmp": "^0.2.4",
"yaml": "^2.2.2"
},
"bin": {
"patch-package": "index.js"
},
"engines": {
"node": ">=14",
"npm": ">5"
}
},
"node_modules/patch-package/node_modules/open": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz",
"integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"is-docker": "^2.0.0",
"is-wsl": "^2.1.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/patch-package/node_modules/slash": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
"integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/patch-package/node_modules/yaml": {
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
"integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
"dev": true,
"license": "ISC",
"bin": {
"yaml": "bin.mjs"
},
"engines": {
"node": ">= 14.6"
},
"funding": {
"url": "https://github.com/sponsors/eemeli"
}
},
"node_modules/path-exists": {
"version": "4.0.0",
"license": "MIT",

View File

@@ -35,15 +35,18 @@
"bump": "node bump-version.js patch",
"bump:minor": "node bump-version.js minor",
"bump:major": "node bump-version.js major",
"bump:patch": "node bump-version.js patch"
"bump:patch": "node bump-version.js patch",
"postinstall": "patch-package"
},
"dependencies": {
"@capacitor-community/speech-recognition": "^7.0.1",
"@capacitor-community/sqlite": "^8.0.0",
"@capacitor/android": "^8.0.0",
"@capacitor/app": "^8.0.0",
"@capacitor/browser": "^8.0.0",
"@capacitor/core": "^8.0.0",
"@capacitor/device": "^8.0.0",
"@capacitor/haptics": "^8.0.2",
"@capacitor/ios": "^8.0.0",
"@capacitor/local-llm": "^1.0.0",
"@capacitor/local-notifications": "^8.0.0",
@@ -124,6 +127,7 @@
"eslint-plugin-sort-keys-fix": "^1.1.2",
"eslint-plugin-tailwindcss": "^3.13.1",
"husky": "^8.0.3",
"patch-package": "^8.0.1",
"postcss": "^8.4.32",
"prettier": "^3.1.1",
"prettier-plugin-tailwindcss": "^0.5.10",

View File

@@ -0,0 +1,26 @@
diff --git a/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java b/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java
index a99ea69..2bea32e 100644
--- a/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java
+++ b/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java
@@ -168,6 +168,7 @@ public class SpeechRecognition extends Plugin implements Constants {
intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, bridge.getActivity().getPackageName());
intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, partialResults);
intent.putExtra("android.speech.extra.DICTATION_MODE", partialResults);
+ intent.putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true);
if (prompt != null) {
intent.putExtra(RecognizerIntent.EXTRA_PROMPT, prompt);
diff --git a/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift b/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift
index 5d1b35b..810277d 100644
--- a/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift
+++ b/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift
@@ -79,6 +79,9 @@ public class SpeechRecognition: CAPPlugin {
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
self.recognitionRequest?.shouldReportPartialResults = partialResults
+ if #available(iOS 13, *), self.speechRecognizer?.supportsOnDeviceRecognition == true {
+ self.recognitionRequest?.requiresOnDeviceRecognition = true
+ }
let inputNode: AVAudioInputNode = self.audioEngine!.inputNode
let format: AVAudioFormat = inputNode.outputFormat(forBus: 0)

View File

@@ -0,0 +1,256 @@
import { Capacitor } from '@capacitor/core'
// Platform-abstracted speech-to-text for voice task capture.
// Native: @capacitor-community/speech-recognition — uses the OS recognizer
// (on-device where the platform supports it, e.g. iOS dictation models).
// Web: Web Speech API (Chrome/Safari) — mainly for development.
//
// Callbacks:
// onPartial(text) — live transcript of the utterance in progress
// onSegment(text) — a finalized utterance (silence/pause boundary)
// onStateChange(bool) — listening started/stopped
// onError(code) — 'denied' | 'error'
//
// Neither OS gives unlimited continuous listening: Android's recognizer ends on
// silence and iOS sessions have a practical duration limit. This service runs a
// restart loop — each recognizer stop commits the buffered utterance as a
// segment and immediately starts a new session while active. The utterance
// boundary doubles as the task boundary.
const SILENCE_COMMIT_MS = 2200
const RESTART_DELAY_MS = 250
const START_OPTIONS = {
language: 'en-US',
maxResults: 1,
partialResults: true,
popup: false,
}
class VoiceInputService {
constructor() {
this._active = false
this._callbacks = null
this._partial = ''
this._lastSpeechAt = 0
this._silenceTimer = null
this._restarting = false
this._webRecognition = null
}
get isNative() {
return Capacitor.isNativePlatform()
}
async isSupported() {
if (this.isNative) {
try {
const { SpeechRecognition } = await import(
'@capacitor-community/speech-recognition'
)
const { available } = await SpeechRecognition.available()
return !!available
} catch {
return false
}
}
return (
typeof window !== 'undefined' &&
!!(window.SpeechRecognition || window.webkitSpeechRecognition)
)
}
async requestPermission() {
if (!this.isNative) {
// Web prompts for the microphone on first start()
return 'granted'
}
try {
const { SpeechRecognition } = await import(
'@capacitor-community/speech-recognition'
)
const current = await SpeechRecognition.checkPermissions()
if (current.speechRecognition === 'granted') return 'granted'
const res = await SpeechRecognition.requestPermissions()
return res.speechRecognition === 'granted' ? 'granted' : 'denied'
} catch {
return 'denied'
}
}
async start(callbacks) {
if (this._active) return
this._callbacks = callbacks
this._active = true
this._partial = ''
this._lastSpeechAt = Date.now()
if (this.isNative) {
await this._startNative()
// Web finalizes utterances itself via isFinal results; only the native
// path needs a silence watchdog to force utterance boundaries.
this._silenceTimer = setInterval(() => this._checkSilence(), 500)
} else {
this._startWeb()
}
this._callbacks?.onStateChange?.(true)
}
async stop() {
if (!this._active) return
this._active = false
if (this._silenceTimer) {
clearInterval(this._silenceTimer)
this._silenceTimer = null
}
if (this.isNative) {
try {
const { SpeechRecognition } = await import(
'@capacitor-community/speech-recognition'
)
await SpeechRecognition.stop()
await SpeechRecognition.removeAllListeners()
} catch {
// recognizer may already be stopped
}
} else if (this._webRecognition) {
const rec = this._webRecognition
this._webRecognition = null
try {
rec.stop()
} catch {
// already stopped
}
}
this._commitPartial()
this._callbacks?.onStateChange?.(false)
}
_commitPartial() {
const text = this._partial.trim()
this._partial = ''
this._callbacks?.onPartial?.('')
if (text) this._callbacks?.onSegment?.(text)
}
_checkSilence() {
if (!this._active || this._restarting) return
if (
this._partial.trim() &&
Date.now() - this._lastSpeechAt > SILENCE_COMMIT_MS
) {
// A pause means the utterance (= task) is complete: cycle the recognizer
// so the buffer commits and a fresh session begins.
this._restartNative()
}
}
async _startNative() {
const { SpeechRecognition } = await import(
'@capacitor-community/speech-recognition'
)
await SpeechRecognition.removeAllListeners()
await SpeechRecognition.addListener('partialResults', ({ matches }) => {
const text = matches?.[0] || ''
if (!text) return
this._partial = text
this._lastSpeechAt = Date.now()
this._callbacks?.onPartial?.(text)
})
await SpeechRecognition.addListener('listeningState', ({ status }) => {
if (status === 'stopped' && this._active && !this._restarting) {
// OS ended the session on its own (silence on Android, session limit
// on iOS) — commit and start over.
this._restartNative()
}
})
// With partialResults the transcript arrives via listeners; the promise's
// resolution/rejection timing differs per platform, so don't rely on it.
SpeechRecognition.start(START_OPTIONS).catch(() => {
if (this._active && !this._restarting) {
this._restartNative()
}
})
}
async _restartNative() {
if (this._restarting) return
this._restarting = true
try {
const { SpeechRecognition } = await import(
'@capacitor-community/speech-recognition'
)
try {
await SpeechRecognition.stop()
} catch {
// already stopped
}
this._commitPartial()
// Let the OS recognizer tear down before starting a new session
await new Promise(r => setTimeout(r, RESTART_DELAY_MS))
if (this._active) {
SpeechRecognition.start(START_OPTIONS).catch(() => {})
this._lastSpeechAt = Date.now()
}
} finally {
this._restarting = false
}
}
_startWeb() {
const SR = window.SpeechRecognition || window.webkitSpeechRecognition
const rec = new SR()
rec.continuous = true
rec.interimResults = true
rec.lang = 'en-US'
rec.onresult = event => {
let interim = ''
for (let i = event.resultIndex; i < event.results.length; i++) {
const res = event.results[i]
if (res.isFinal) {
this._partial = ''
this._callbacks?.onPartial?.('')
const text = res[0].transcript.trim()
if (text) this._callbacks?.onSegment?.(text)
} else {
interim += res[0].transcript
}
}
if (interim) {
this._partial = interim
this._lastSpeechAt = Date.now()
this._callbacks?.onPartial?.(interim)
}
}
rec.onend = () => {
if (this._active && this._webRecognition === rec) {
try {
rec.start()
} catch {
// restart can race with teardown
}
}
}
rec.onerror = e => {
if (e.error === 'not-allowed' || e.error === 'service-not-allowed') {
this._callbacks?.onError?.('denied')
this.stop()
}
}
this._webRecognition = rec
try {
rec.start()
} catch {
this._callbacks?.onError?.('error')
}
}
}
export const voiceInputService = new VoiceInputService()

View File

@@ -24,6 +24,7 @@ import SmartTaskTitleInput from './SmartTaskTitleInput'
import KeyboardShortcutHint from '../../components/common/KeyboardShortcutHint'
import { useDocumentScanner } from '../../hooks/useDocumentScanner'
import { localAIService } from '../../service/LocalAIService'
import { voiceInputService } from '../../service/VoiceInputService'
import { TASK_COLOR } from '../../utils/Colors'
import AdvancedOptionsSection, {
AdvancedOptionsTrigger,
@@ -39,6 +40,8 @@ import RepeatPickerField from './RepeatPickerField'
import RichTextEditor from './RichTextEditor'
import ScanPanel from './ScanToTask/ScanPanel'
import SubTasks from './SubTask'
import { buildChorePayload } from './VoiceToTask/parseVoiceTask'
import VoicePanel from './VoiceToTask/VoicePanel'
const getDefaultNotification = () => {
const storedDefault = localStorage.getItem('defaultNotificationTemplate')
if (storedDefault) {
@@ -121,10 +124,13 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
const [scanAutoCapture, setScanAutoCapture] = useState(false)
const [pendingPhotoUrl, setPendingPhotoUrl] = useState(null)
const [llmAvailable, setLlmAvailable] = useState(false)
const [showVoice, setShowVoice] = useState(false)
const [voiceAvailable, setVoiceAvailable] = useState(false)
const { isNativeScanner } = useDocumentScanner()
useEffect(() => {
localAIService.isAvailable().then(setLlmAvailable)
voiceInputService.isSupported().then(setVoiceAvailable)
}, [])
// Priority colors
@@ -589,9 +595,46 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
}
}
// Single voice-captured task: land it in the smart input so the user
// reviews it with the normal pickers before creating.
const handleVoiceSingle = text => {
setShowVoice(false)
processText(text)
}
// Multiple voice-captured tasks: they were reviewed as cards in the panel,
// so create them all directly.
const handleVoiceCreateMany = async parsedTasks => {
const notificationTemplates = getDefaultNotification()
for (const parsed of parsedTasks) {
const chore = buildChorePayload(parsed, {
userProfile,
projectId,
notificationTemplates,
})
try {
const result = await createChoreMutation.mutateAsync(chore)
if (result?._pendingCreate) {
onChoreUpdate(result)
} else {
onChoreUpdate({
...chore,
...result,
id: result?.id,
nextDueDate: chore.dueDate,
})
}
} catch (error) {
console.error('Error creating voice task:', error)
}
}
handleCloseModal(false)
}
const handleCloseModal = forceRefetch => {
onClose(forceRefetch)
setShowScan(false)
setShowVoice(false)
setTaskText('')
setTaskTitle('')
setDueDate(null)
@@ -761,7 +804,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
</Box>
}
>
{!showScan && (
{!showScan && !showVoice && (
<>
<Box>
<Box
@@ -838,6 +881,9 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
}
: undefined
}
onVoiceClick={
voiceAvailable ? () => setShowVoice(true) : undefined
}
placeholder='Type your task...'
onChange={text => {
setTaskText(text)
@@ -1097,6 +1143,18 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => {
</>
)}
{showVoice && (
<VoicePanel
open
userLabels={userLabels || []}
members={circleMembers?.res || []}
userProfile={userProfile}
onClose={() => setShowVoice(false)}
onUseSingle={handleVoiceSingle}
onCreateMany={handleVoiceCreateMany}
/>
)}
{showScan && (
<ScanPanel
open

View File

@@ -1,4 +1,4 @@
import { CameraEnhance, PhotoFilter } from '@mui/icons-material'
import { CameraEnhance, Mic, PhotoFilter } from '@mui/icons-material'
import { IconButton, Tooltip, useColorScheme } from '@mui/joy'
import { useEffect, useRef, useState } from 'react'
import AutocompleteDropdown from '../TestView/AutocompleteDropdown'
@@ -57,6 +57,7 @@ const SmartTaskTitleInput = ({
isNativeScanner,
onScanClick,
onPhotoSelected,
onVoiceClick,
}) => {
const { mode, setMode } = useColorScheme()
const titleInputRef = useRef(null)
@@ -200,13 +201,14 @@ const SmartTaskTitleInput = ({
e.target.value = ''
}
const showNativeButtons = isNativeScanner && !value
const MIC_BUTTON_WIDTH =
showNativeButtons && onPhotoSelected && onScanClick
? '5rem'
: showNativeButtons
? '2.5rem'
: '0rem'
const showPhotoButtons = isNativeScanner && !value
const showVoiceButton = !!onVoiceClick && !value
const visibleButtonCount =
(showPhotoButtons && onPhotoSelected ? 1 : 0) +
(showPhotoButtons && onScanClick ? 1 : 0) +
(showVoiceButton ? 1 : 0)
const showActionButtons = visibleButtonCount > 0
const ACTION_BUTTONS_WIDTH = `${visibleButtonCount * 2.5}rem`
return (
<div>
@@ -223,7 +225,7 @@ const SmartTaskTitleInput = ({
position: 'absolute',
top: 0,
left: 0,
width: `calc(100% - ${MIC_BUTTON_WIDTH})`,
width: `calc(100% - ${ACTION_BUTTONS_WIDTH})`,
height: '100%',
zIndex: 1,
resize: 'none',
@@ -274,7 +276,7 @@ const SmartTaskTitleInput = ({
{/* Zero-width space to maintain consistent height */}
&#8203;
</div>
{showNativeButtons && (
{showActionButtons && (
<span
style={{
position: 'absolute',
@@ -286,7 +288,7 @@ const SmartTaskTitleInput = ({
gap: '0.1rem',
}}
>
{onPhotoSelected && (
{showPhotoButtons && onPhotoSelected && (
<>
<Tooltip title='Select photo' placement='top' size='sm'>
<IconButton
@@ -308,7 +310,7 @@ const SmartTaskTitleInput = ({
/>
</>
)}
{onScanClick && (
{showPhotoButtons && onScanClick && (
<Tooltip title='Scan to create task' placement='top' size='sm'>
<IconButton
size='sm'
@@ -321,6 +323,19 @@ const SmartTaskTitleInput = ({
</IconButton>
</Tooltip>
)}
{showVoiceButton && (
<Tooltip title='Speak to create tasks' placement='top' size='sm'>
<IconButton
size='sm'
variant='plain'
color='neutral'
onClick={onVoiceClick}
sx={{ borderRadius: 'xl' }}
>
<Mic fontSize='small' />
</IconButton>
</Tooltip>
)}
</span>
)}
</div>

View File

@@ -0,0 +1,121 @@
.voice-mic-btn {
position: relative;
width: 72px;
height: 72px;
border-radius: 50%;
border: none;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
background: var(--joy-palette-primary-solidBg, #0b6bcb);
color: #fff;
transition:
transform 0.15s ease,
background 0.2s ease,
box-shadow 0.2s ease;
touch-action: none;
user-select: none;
-webkit-user-select: none;
-webkit-tap-highlight-color: transparent;
}
.voice-mic-btn:active {
transform: scale(0.94);
}
.voice-mic-btn.listening {
background: var(--joy-palette-danger-solidBg, #c41c1c);
box-shadow: 0 4px 18px rgba(196, 28, 28, 0.35);
}
.voice-pulse-ring {
position: absolute;
inset: 0;
border-radius: 50%;
pointer-events: none;
opacity: 0;
}
.voice-mic-btn.listening .voice-pulse-ring {
opacity: 1;
animation: voice-pulse 1.8s ease-out infinite;
}
.voice-mic-btn.listening .voice-pulse-ring:nth-child(2) {
animation-delay: 0.6s;
}
@keyframes voice-pulse {
0% {
box-shadow: 0 0 0 0 rgba(196, 28, 28, 0.4);
}
100% {
box-shadow: 0 0 0 26px rgba(196, 28, 28, 0);
}
}
/* Faux equalizer shown while listening */
.voice-eq {
display: flex;
gap: 3px;
align-items: center;
height: 28px;
}
.voice-eq span {
width: 4px;
border-radius: 2px;
background: var(--joy-palette-danger-solidBg, #c41c1c);
animation: voice-eq-wave 1.1s ease-in-out infinite;
}
.voice-eq span:nth-child(1) {
animation-delay: 0s;
}
.voice-eq span:nth-child(2) {
animation-delay: 0.18s;
}
.voice-eq span:nth-child(3) {
animation-delay: 0.32s;
}
.voice-eq span:nth-child(4) {
animation-delay: 0.12s;
}
.voice-eq span:nth-child(5) {
animation-delay: 0.26s;
}
@keyframes voice-eq-wave {
0%,
100% {
height: 6px;
}
50% {
height: 24px;
}
}
/* Committed task cards slide in as segments are captured */
.voice-task-card {
animation: voice-card-in 0.25s ease-out;
}
@keyframes voice-card-in {
from {
opacity: 0;
transform: translateY(6px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
@media (prefers-reduced-motion: reduce) {
.voice-mic-btn.listening .voice-pulse-ring,
.voice-eq span,
.voice-task-card {
animation: none;
}
}

View File

@@ -0,0 +1,493 @@
import {
CalendarMonth,
Close,
Flag,
GraphicEq,
Lock,
Mic,
Person,
Repeat,
Sell,
Toll,
WarningAmber,
} from '@mui/icons-material'
import { Box, Button, Chip, IconButton, Input, Typography } from '@mui/joy'
import moment from 'moment'
import { useMemo, useState } from 'react'
import { parseVoiceTask } from './parseVoiceTask'
import { useVoiceToTask } from './useVoiceToTask'
import './VoicePanel.css'
const HIGHLIGHT_CLASS = {
repeat: 'highlight-repeat',
priority: 'highlight-priority',
points: 'highlight-points',
assignee: 'highlight-assignee',
label: 'highlight-label',
dueDate: 'highlight-date',
}
const renderTranscript = (text, highlights) => {
const parts = []
let lastIndex = 0
for (const h of highlights) {
if (h.start > lastIndex) parts.push(text.substring(lastIndex, h.start))
parts.push(
<span
key={h.start}
className={HIGHLIGHT_CLASS[h.type]}
style={{
textDecoration: 'underline',
textDecorationThickness: '2px',
textDecorationStyle: 'dashed',
}}
>
{text.substring(h.start, h.end)}
</span>,
)
lastIndex = h.end
}
if (lastIndex < text.length) parts.push(text.substring(lastIndex))
return parts
}
const formatDue = dueDate => {
const m = moment(dueDate)
return m.format('HH:mm') === '23:59'
? m.format('MMM D')
: m.format('MMM D, h:mm A')
}
const buildChips = (parsed, { members, currentUserId }) => {
const chips = []
if (parsed.dueDate) {
chips.push({
key: 'due',
color: 'warning',
icon: <CalendarMonth sx={{ fontSize: 12 }} />,
label: formatDue(parsed.dueDate),
})
}
if (parsed.frequencyName) {
chips.push({
key: 'repeat',
color: 'success',
icon: <Repeat sx={{ fontSize: 12 }} />,
label: parsed.frequencyName,
})
}
if (parsed.priority > 0) {
chips.push({
key: 'priority',
color: 'danger',
icon: <Flag sx={{ fontSize: 12 }} />,
label: `P${parsed.priority}`,
})
}
if (parsed.points != null) {
chips.push({
key: 'points',
color: 'primary',
icon: <Toll sx={{ fontSize: 12 }} />,
label: `${parsed.points} pts`,
})
}
parsed.labelNames.forEach(name => {
chips.push({
key: `label-${name}`,
color: 'primary',
icon: <Sell sx={{ fontSize: 12 }} />,
label: name,
})
})
if (parsed.isAnyone) {
chips.push({
key: 'assignee',
color: 'neutral',
icon: <Person sx={{ fontSize: 12 }} />,
label: 'Anyone',
})
} else if (
parsed.assignees.length > 0 &&
parsed.assignees[0].userId !== currentUserId
) {
const member = members.find(m => m.userId === parsed.assignees[0].userId)
if (member) {
chips.push({
key: 'assignee',
color: 'neutral',
icon: <Person sx={{ fontSize: 12 }} />,
label: member.displayName,
})
}
}
return chips
}
const TaskPreviewCard = ({ segment, parseCtx, onRemove, onUpdate }) => {
const [editing, setEditing] = useState(false)
const [draft, setDraft] = useState(segment.text)
const parsed = useMemo(
() => parseVoiceTask(segment.text, parseCtx),
[segment.text, parseCtx],
)
const chips = useMemo(() => buildChips(parsed, parseCtx), [parsed, parseCtx])
const commitEdit = () => {
setEditing(false)
if (draft.trim() !== segment.text) onUpdate(draft)
}
return (
<Box
className='voice-task-card'
sx={{
borderRadius: 'md',
border: '1px solid',
borderColor: 'divider',
bgcolor: 'background.surface',
p: 1.25,
display: 'flex',
flexDirection: 'column',
gap: 0.75,
}}
>
<Box sx={{ display: 'flex', alignItems: 'flex-start', gap: 1 }}>
{editing ? (
<Input
size='sm'
autoFocus
value={draft}
onChange={e => setDraft(e.target.value)}
onKeyDown={e => {
if (e.key === 'Enter') commitEdit()
if (e.key === 'Escape') {
setDraft(segment.text)
setEditing(false)
}
}}
onBlur={commitEdit}
sx={{ flex: 1 }}
/>
) : (
<Typography
level='title-sm'
sx={{ flex: 1, cursor: 'text', wordBreak: 'break-word' }}
onClick={() => {
setDraft(segment.text)
setEditing(true)
}}
>
{parsed.title || segment.text}
</Typography>
)}
<IconButton
size='sm'
variant='plain'
color='neutral'
onClick={onRemove}
sx={{ '--IconButton-size': '28px' }}
>
<Close fontSize='small' />
</IconButton>
</Box>
{chips.length > 0 && (
<Box sx={{ display: 'flex', flexWrap: 'wrap', gap: 0.5 }}>
{chips.map(chip => (
<Chip
key={chip.key}
size='sm'
variant='soft'
color={chip.color}
startDecorator={chip.icon}
>
{chip.label}
</Chip>
))}
</Box>
)}
</Box>
)
}
/**
* Inline voice-to-task panel. Mounts inside AddTaskModal — no second modal.
*
* Hold the mic to speak, or tap once for hands-free. Pauses and spoken
* separators ("also") split the transcript into task cards. A single captured
* task lands in the smart input for review; multiple tasks are created
* directly from the review list.
*/
const VoicePanel = ({
open,
userLabels = [],
members = [],
userProfile,
onClose,
onUseSingle,
onCreateMany,
}) => {
const {
phase,
isLocked,
partialText,
segments,
micPressDown,
micPressUp,
startListening,
removeSegment,
updateSegment,
reset,
isNative,
} = useVoiceToTask({ members })
const [creating, setCreating] = useState(false)
const parseCtx = useMemo(
() => ({ userLabels, members, currentUserId: userProfile?.id }),
[userLabels, members, userProfile?.id],
)
const partialParsed = useMemo(
() => (partialText ? parseVoiceTask(partialText, parseCtx) : null),
[partialText, parseCtx],
)
if (!open) return null
const isListening = phase === 'listening'
const showActions = segments.length > 0 && !isListening && !creating
const handleCancel = () => {
reset()
onClose()
}
const handleCreateAll = async () => {
setCreating(true)
try {
await onCreateMany(segments.map(s => parseVoiceTask(s.text, parseCtx)))
} finally {
setCreating(false)
}
}
const micCaption = isListening
? isLocked
? 'Listening — tap to stop'
: 'Release to finish · quick tap locks hands-free'
: segments.length > 0
? 'Hold to add another task'
: 'Hold to speak · quick tap for hands-free'
return (
<Box
sx={{
borderRadius: 'md',
border: '1px solid',
borderColor: 'primary.outlinedBorder',
overflow: 'hidden',
bgcolor: 'background.level1',
}}
>
{/* ── Header ── */}
<Box
sx={{
px: 1.5,
pt: 1.25,
display: 'flex',
alignItems: 'center',
gap: 1,
}}
>
<GraphicEq color='primary' fontSize='small' />
<Typography level='title-sm'>Speak your tasks</Typography>
{isNative && (
<Chip
size='sm'
variant='soft'
color='success'
startDecorator={<Lock sx={{ fontSize: 12 }} />}
sx={{ ml: 'auto' }}
>
On-device
</Chip>
)}
</Box>
{/* ── Permission denied ── */}
{phase === 'denied' && (
<Box sx={{ p: 2 }}>
<Box
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1, mb: 1.5 }}
>
<WarningAmber color='warning' sx={{ mt: 0.25, flexShrink: 0 }} />
<Typography level='body-sm'>
Microphone access is needed for voice capture. Enable it in your
device settings and try again.
</Typography>
</Box>
<Button
size='sm'
variant='outlined'
color='neutral'
onClick={startListening}
>
Try Again
</Button>
</Box>
)}
{/* ── Captured task cards ── */}
{segments.length > 0 && (
<Box
sx={{
px: 1.5,
pt: 1.25,
display: 'flex',
flexDirection: 'column',
gap: 0.75,
maxHeight: 260,
overflowY: 'auto',
}}
>
{segments.map(segment => (
<TaskPreviewCard
key={segment.id}
segment={segment}
parseCtx={parseCtx}
onRemove={() => removeSegment(segment.id)}
onUpdate={text => updateSegment(segment.id, text)}
/>
))}
</Box>
)}
{/* ── Live transcript ── */}
{isListening && (
<Box sx={{ px: 1.5, pt: 1.25 }}>
<Box
sx={{
minHeight: 44,
borderRadius: 'md',
border: '1px dashed',
borderColor: 'neutral.outlinedBorder',
bgcolor: 'background.surface',
px: 1.25,
py: 1,
}}
>
{partialText ? (
<Typography level='body-md' sx={{ wordBreak: 'break-word' }}>
{renderTranscript(partialText, partialParsed?.highlights || [])}
</Typography>
) : (
<Typography level='body-sm' sx={{ opacity: 0.5 }}>
Listening
</Typography>
)}
</Box>
</Box>
)}
{/* ── Mic stage ── */}
{phase !== 'denied' && (
<Box
sx={{
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
py: 2,
gap: 1.25,
}}
>
<div className='voice-eq' style={{ opacity: isListening ? 1 : 0 }}>
<span />
<span />
<span />
<span />
<span />
</div>
<button
type='button'
aria-label={isListening ? 'Stop listening' : 'Start voice capture'}
className={`voice-mic-btn${isListening ? ' listening' : ''}`}
onPointerDown={e => {
e.preventDefault()
e.currentTarget.setPointerCapture?.(e.pointerId)
micPressDown()
}}
onPointerUp={micPressUp}
onPointerCancel={micPressUp}
onContextMenu={e => e.preventDefault()}
>
<span className='voice-pulse-ring' />
<span className='voice-pulse-ring' />
<Mic sx={{ fontSize: 32 }} />
</button>
<Typography level='body-xs' sx={{ opacity: 0.7 }}>
{micCaption}
</Typography>
<Typography
level='body-xs'
sx={{ opacity: 0.5, px: 2, textAlign: 'center' }}
>
Pause or say &ldquo;also&rdquo; between tasks &middot; say
&ldquo;scratch that&rdquo; to remove the last one
</Typography>
</Box>
)}
{/* ── Footer ── */}
<Box
sx={{
px: 1.5,
py: 1,
display: 'flex',
alignItems: 'center',
gap: 1,
borderTop: '1px solid',
borderColor: 'divider',
}}
>
<Button
size='sm'
variant='plain'
color='neutral'
onClick={handleCancel}
>
Cancel
</Button>
<Box sx={{ ml: 'auto', display: 'flex', gap: 1 }}>
{creating && (
<Button size='sm' variant='solid' color='primary' loading>
Creating
</Button>
)}
{showActions &&
(segments.length === 1 ? (
<Button
size='sm'
variant='solid'
color='primary'
onClick={() => onUseSingle(segments[0].text)}
>
Use Task
</Button>
) : (
<Button
size='sm'
variant='solid'
color='primary'
onClick={handleCreateAll}
>
Create {segments.length} Tasks
</Button>
))}
</Box>
</Box>
</Box>
)
}
export default VoicePanel

View File

@@ -0,0 +1,201 @@
import * as chrono from 'chrono-node'
import moment from 'moment'
import { isPlusAccount } from '../../../utils/Helpers'
import { generateUUID } from '../../../utils/UUID'
import {
parseAssignees,
parseDueDate,
parseLabels,
parsePoints,
parsePriority,
parseRepeatV2,
} from '../CustomParsers'
// Pure equivalent of AddTaskModal.processText — parses one sentence into a
// structured task (no state setters), preserving the same parser order and
// sequential-cleanup behavior so voice and typed input stay consistent.
const mapMembersForParsing = members =>
members.map(member => ({
userId: member.userId,
username:
member.username || member.displayName?.toLowerCase().replace(/\s+/g, ''),
displayName: member.displayName,
name: member.displayName,
id: member.userId,
}))
// Merge overlapping highlight ranges, higher parser priority wins — same
// resolution rules as AddTaskModal.renderHighlightedSentence.
const resolveHighlights = ({
repeat,
priority,
points,
assignees,
labels,
dueDate,
}) => {
const all = []
repeat?.forEach(h => all.push({ ...h, type: 'repeat', rank: 60 }))
priority?.forEach(h => all.push({ ...h, type: 'priority', rank: 50 }))
points?.forEach(h => all.push({ ...h, type: 'points', rank: 45 }))
assignees?.forEach(h => all.push({ ...h, type: 'assignee', rank: 40 }))
labels?.forEach(h => all.push({ ...h, type: 'label', rank: 30 }))
if (dueDate) all.push({ ...dueDate, type: 'dueDate', rank: 20 })
all.sort((a, b) => a.start - b.start)
const resolved = []
for (const current of all) {
const previous = resolved[resolved.length - 1]
if (previous && current.start < previous.end) {
if (current.rank > previous.rank) {
resolved.pop()
resolved.push(current)
}
} else {
resolved.push(current)
}
}
return resolved
}
export const parseVoiceTask = (
sentence,
{ userLabels = [], members = [], currentUserId = null } = {},
) => {
const assigneesForParsing = mapMembersForParsing(members)
const priority = parsePriority(sentence)
const points = parsePoints(sentence)
const labels = parseLabels(sentence, userLabels)
const assigneesResult = parseAssignees(sentence, assigneesForParsing)
const repeat = parseRepeatV2(sentence)
const dueDateParsed = parseDueDate(sentence, chrono)
// Sequential cleanup — identical chain to AddTaskModal.processText
let cleaned = sentence
if (priority.result) cleaned = priority.cleanedSentence
if (points.result) {
const reparse = parsePoints(cleaned)
if (reparse.result) cleaned = reparse.cleanedSentence
}
if (labels.result) {
const reparse = parseLabels(cleaned, userLabels)
if (reparse.result) cleaned = reparse.cleanedSentence
}
if (assigneesResult.result) {
const reparse = parseAssignees(cleaned, assigneesForParsing)
if (reparse.result) cleaned = reparse.cleanedSentence
}
if (repeat.result) {
const reparse = parseRepeatV2(cleaned)
if (reparse.result) cleaned = reparse.cleanedSentence
}
if (dueDateParsed.result) {
const reparse = parseDueDate(cleaned, chrono)
if (reparse.result) cleaned = reparse.cleanedSentence
}
let dueDate = null
if (dueDateParsed.result) {
dueDate = moment(dueDateParsed.result).format('YYYY-MM-DDTHH:mm:ss')
} else if (repeat.dueDate) {
dueDate = moment(repeat.dueDate).format('YYYY-MM-DDTHH:mm:ss')
}
let assignees = []
const isAnyone = !!assigneesResult.isAnyone
if (!isAnyone) {
if (assigneesResult.result?.length > 0) {
assignees = assigneesResult.result.map(a => ({ userId: a.userId }))
} else if (currentUserId) {
assignees = [{ userId: currentUserId }]
}
}
const labelIds = (labels.result || [])
.filter(label => label.id)
.map(label => label.id)
return {
raw: sentence,
title: cleaned.replace(/\s+/g, ' ').trim(),
priority: priority.result ? parseInt(priority.result, 10) : 0,
points: points.result ?? null,
labelIds,
labelNames: (labels.result || []).map(label => label.name),
assignees,
isAnyone,
frequency: repeat.result,
frequencyName: repeat.name,
dueDate,
highlights: resolveHighlights({
repeat: repeat.highlight,
priority: priority.highlight,
points: points.highlight,
assignees: assigneesResult.highlight,
labels: labels.highlight,
dueDate: dueDateParsed.result ? dueDateParsed.highlight[0] : null,
}),
}
}
// Builds the same chore payload shape AddTaskModal.createChore submits.
export const buildChorePayload = (
parsed,
{ userProfile, projectId, notificationTemplates },
) => {
let finalAssignees = parsed.assignees
let finalAssignedTo = null
let finalAssignStrategy = 'keep_last_assigned'
if (parsed.isAnyone) {
finalAssignees = []
finalAssignStrategy = 'no_assignee'
} else if (finalAssignees.length === 0) {
finalAssignees = [{ userId: userProfile?.id }]
finalAssignedTo = userProfile?.id
} else {
finalAssignedTo = finalAssignees[0].userId
}
const chore = {
name: parsed.title,
description: null,
assignees: finalAssignees,
dueDate: parsed.dueDate ? new Date(parsed.dueDate).toISOString() : null,
assignedTo: finalAssignedTo,
assignStrategy: finalAssignStrategy,
isRolling: false,
labelsV2: parsed.labelIds,
priority: parsed.priority || 0,
points: parsed.points ?? null,
deadlineOffset: null,
completionWindow: null,
requireApproval: false,
isPrivate: false,
status: 0,
frequencyType: 'once',
frequencyMetadata: {},
notificationMetadata: {},
subTasks: null,
projectId: projectId === 'default' ? null : projectId,
draftId: generateUUID(),
}
if (parsed.frequency) {
chore.frequencyType = parsed.frequency.frequencyType
chore.frequencyMetadata = parsed.frequency.frequencyMetadata
chore.frequency = parsed.frequency.frequency
if (isPlusAccount(userProfile)) {
chore.notification = true
chore.notificationMetadata = { templates: notificationTemplates }
}
}
if (!parsed.frequency && parsed.dueDate) {
chore.nextDueDate = new Date(parsed.dueDate).toISOString()
chore.notificationMetadata = { templates: notificationTemplates }
}
return chore
}

View File

@@ -0,0 +1,217 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { voiceInputService } from '../../../service/VoiceInputService'
import { generateUUID } from '../../../utils/UUID'
import {
applyScratchThat,
normalizeSpokenText,
splitSpokenSegments,
} from './voiceNormalizer'
// Mic gesture: hold = push-to-talk (release stops), quick tap = hands-free
// lock (tap again to stop). In hands-free mode, sustained silence auto-stops
// into review so the user is never stuck watching a live mic.
const TAP_THRESHOLD_MS = 400
const HANDS_FREE_SILENCE_STOP_MS = 8000
const HANDS_FREE_EMPTY_STOP_MS = 20000
const haptic = async kind => {
try {
const { Haptics, ImpactStyle, NotificationType } = await import(
'@capacitor/haptics'
)
if (kind === 'notification') {
await Haptics.notification({ type: NotificationType.Success })
} else if (kind === 'medium') {
await Haptics.impact({ style: ImpactStyle.Medium })
} else {
await Haptics.impact({ style: ImpactStyle.Light })
}
} catch {
// no haptics on this platform
}
}
// phases: idle | listening | review | denied
export function useVoiceToTask({ members = [] } = {}) {
const [phase, setPhase] = useState('idle')
const [isLocked, setIsLocked] = useState(false)
const [partialText, setPartialText] = useState('')
const [segments, setSegments] = useState([])
// Kept in sync manually (not via render) so segment commits that happen
// inside voiceInputService.stop() are visible immediately afterwards.
const segmentsRef = useRef([])
const membersRef = useRef(members)
membersRef.current = members
const phaseRef = useRef(phase)
phaseRef.current = phase
const lockedRef = useRef(isLocked)
lockedRef.current = isLocked
const pressStartedAtRef = useRef(0)
const pressStartedListeningRef = useRef(false)
const lastActivityRef = useRef(0)
const watchdogRef = useRef(null)
const applySegments = useCallback(next => {
segmentsRef.current = next
setSegments(next)
}, [])
const commitSegment = useCallback(
rawText => {
const normalized = normalizeSpokenText(rawText, {
members: membersRef.current,
})
const { text, dropPrevious } = applyScratchThat(normalized)
const pieces = splitSpokenSegments(text)
if (!dropPrevious && pieces.length === 0) return
let base = segmentsRef.current
if (dropPrevious && base.length > 0) {
base = base.slice(0, -1)
haptic('medium')
}
if (pieces.length > 0) haptic('light')
applySegments([
...base,
...pieces.map(piece => ({ id: generateUUID(), text: piece })),
])
},
[applySegments],
)
const stopListening = useCallback(async () => {
if (watchdogRef.current) {
clearInterval(watchdogRef.current)
watchdogRef.current = null
}
await voiceInputService.stop()
setPartialText('')
setIsLocked(false)
// stop() commits any buffered partial synchronously through onSegment,
// so the ref is up to date by the time we read it
setPhase(segmentsRef.current.length > 0 ? 'review' : 'idle')
haptic('light')
}, [])
const startListening = useCallback(async () => {
const permission = await voiceInputService.requestPermission()
if (permission !== 'granted') {
setPhase('denied')
return false
}
lastActivityRef.current = Date.now()
await voiceInputService.start({
onPartial: text => {
lastActivityRef.current = Date.now()
setPartialText(
normalizeSpokenText(text, { members: membersRef.current }),
)
},
onSegment: commitSegment,
onError: () => {
setPhase('denied')
},
onStateChange: () => {},
})
setPhase('listening')
haptic('medium')
// Hands-free: auto-stop into review after sustained silence
watchdogRef.current = setInterval(() => {
if (phaseRef.current !== 'listening' || !lockedRef.current) return
const idleFor = Date.now() - lastActivityRef.current
const limit =
segmentsRef.current.length > 0
? HANDS_FREE_SILENCE_STOP_MS
: HANDS_FREE_EMPTY_STOP_MS
if (idleFor > limit) {
stopListening()
}
}, 1000)
return true
}, [commitSegment, stopListening])
const micPressDown = useCallback(() => {
pressStartedAtRef.current = Date.now()
if (phaseRef.current === 'listening') {
pressStartedListeningRef.current = false
return
}
pressStartedListeningRef.current = true
startListening()
}, [startListening])
const micPressUp = useCallback(() => {
const held = Date.now() - pressStartedAtRef.current
if (pressStartedListeningRef.current) {
if (held < TAP_THRESHOLD_MS) {
// Quick tap → hands-free lock
setIsLocked(true)
} else {
// Hold-to-talk → release ends the capture
stopListening()
}
} else if (phaseRef.current === 'listening') {
// Tap while already listening (locked mode) → stop
stopListening()
}
pressStartedListeningRef.current = false
}, [stopListening])
const removeSegment = useCallback(
id => {
applySegments(segmentsRef.current.filter(s => s.id !== id))
},
[applySegments],
)
const updateSegment = useCallback(
(id, text) => {
applySegments(
segmentsRef.current
.map(s => (s.id === id ? { ...s, text: text.trim() } : s))
.filter(s => s.text),
)
},
[applySegments],
)
const reset = useCallback(() => {
voiceInputService.stop()
if (watchdogRef.current) {
clearInterval(watchdogRef.current)
watchdogRef.current = null
}
applySegments([])
setPartialText('')
setIsLocked(false)
setPhase('idle')
}, [applySegments])
// Stop the recognizer if the panel unmounts mid-capture
useEffect(() => {
return () => {
voiceInputService.stop()
if (watchdogRef.current) clearInterval(watchdogRef.current)
}
}, [])
return {
phase,
isLocked,
partialText,
segments,
micPressDown,
micPressUp,
startListening,
stopListening,
removeSegment,
updateSegment,
reset,
isNative: voiceInputService.isNative,
}
}

View File

@@ -0,0 +1,129 @@
// Deterministic transforms that turn spoken language into the typed syntax
// CustomParsers understands. No LLM — instant, predictable, fully offline.
//
// "label groceries" → "#groceries"
// "assign to Sarah" → "@Sarah" (only when Sarah is a circle member)
// "worth five points" → "*5"
// "p one" / "top priority" → "priority 1" (parsePriority already handles that)
const FILLER_REGEX = /(?:^|\s)(?:um+|uh+|erm+|hmm+|mmm+)(?=[\s,.!?]|$)[,.]?/gi
const NUMBER_WORDS = {
one: 1,
two: 2,
three: 3,
four: 4,
five: 5,
six: 6,
seven: 7,
eight: 8,
nine: 9,
ten: 10,
fifteen: 15,
twenty: 20,
'twenty five': 25,
'twenty-five': 25,
fifty: 50,
hundred: 100,
'one hundred': 100,
}
const NUMBER_WORD_PATTERN = Object.keys(NUMBER_WORDS)
// Longest first so "twenty five" wins over "five"
.sort((a, b) => b.length - a.length)
.join('|')
const escapeRegex = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
export const stripFillers = text =>
text.replace(FILLER_REGEX, ' ').replace(/\s+/g, ' ').trim()
const normalizePriority = text =>
text
.replace(/\b(?:top|highest)\s+priority\b/gi, 'priority 1')
.replace(
/\bp[\s-]?(one|two|three|four|[1-4])\b/gi,
(_, n) => `priority ${NUMBER_WORDS[n.toLowerCase()] || n}`,
)
const normalizePoints = text =>
text.replace(
new RegExp(
`\\b(?:worth\\s+)?(\\d+|${NUMBER_WORD_PATTERN})\\s+points?\\b`,
'gi',
),
(_, n) => `*${NUMBER_WORDS[n.toLowerCase()] || n} points`,
)
const normalizeLabels = text =>
text.replace(
/\b(?:with\s+)?(?:hash\s?tag|labell?ed(?:\s+as)?|label|tagged(?:\s+as)?|tag)\s+([\p{L}\p{N}_]+)/giu,
'#$1',
)
const normalizeAssignees = (text, members = []) => {
const assignVerb = '(?:assign(?:ed)?\\s+(?:this\\s+|it\\s+)?to|for)'
let out = text.replace(
new RegExp(`\\b${assignVerb}\\s+(?:anyone|anybody|everyone)\\b`, 'gi'),
'@Anyone',
)
for (const member of members) {
const displayName = member.displayName
if (!displayName) continue
const firstName = displayName.split(/\s+/)[0]
// Full display name first so "assign to Mo Tarbin" doesn't leave "Tarbin"
const names = [...new Set([displayName, firstName])].filter(
n => n.length > 1,
)
for (const name of names) {
out = out.replace(
new RegExp(`\\b${assignVerb}\\s+${escapeRegex(name)}\\b`, 'gi'),
`@${displayName}`,
)
}
}
return out
}
export const normalizeSpokenText = (text, { members = [] } = {}) => {
let out = stripFillers(text)
out = normalizePriority(out)
out = normalizePoints(out)
out = normalizeLabels(out)
out = normalizeAssignees(out, members)
return out.replace(/\s+/g, ' ').trim()
}
// ── Multi-task segmentation ─────────────────────────────────────────────────
// A pause (utterance boundary) always splits — that's handled upstream by the
// recognizer. These spoken separators additionally split within one utterance.
// Deliberately conservative: "and then" is NOT a separator ("wash and then
// fold laundry" is one task).
const SEPARATOR_REGEX =
/\s*\b(?:and\s+also|also|next\s+task|new\s+task|another\s+task)\b[,.]?\s*/gi
export const splitSpokenSegments = text =>
text
.split(SEPARATOR_REGEX)
.map(s => s.trim().replace(/^[,.]\s*/, ''))
.filter(Boolean)
// ── "Scratch that" correction ───────────────────────────────────────────────
// Everything spoken before the command dies. If the command opens the
// utterance ("…pause… scratch that"), the previously committed task dies
// instead. Words after the command carry on as the replacement.
const SCRATCH_REGEX =
/\s*\b(?:(?:scratch|forget|delete|remove|cancel)\s+(?:that|this|it|last(?:\s+one)?)|never\s?mind)\b[,.]?\s*/gi
export const applyScratchThat = text => {
const parts = text.split(SCRATCH_REGEX)
if (parts.length === 1) {
return { text: text.trim(), dropPrevious: false }
}
const before = parts.slice(0, -1).join(' ').trim()
const after = parts[parts.length - 1].trim()
return { text: after, dropPrevious: before.length === 0 }
}