From e0257445efe50b897a98ae5f57f0ca9818530622 Mon Sep 17 00:00:00 2001 From: Mo Tarbin Date: Mon, 20 Jul 2026 01:41:42 -0400 Subject: [PATCH] Add Initial Voice Input --- android/app/capacitor.build.gradle | 2 + android/app/src/main/AndroidManifest.xml | 9 + android/capacitor.settings.gradle | 6 + ios/App/App/Info.plist | 4 + ios/App/Podfile | 2 + ios/App/Podfile.lock | 14 +- package-lock.json | 179 ++++++- package.json | 6 +- ...r-community+speech-recognition+7.0.1.patch | 26 + src/service/VoiceInputService.js | 256 +++++++++ src/views/components/AddTaskModal.jsx | 60 ++- src/views/components/SmartTaskTitleInput.jsx | 39 +- .../components/VoiceToTask/VoicePanel.css | 121 +++++ .../components/VoiceToTask/VoicePanel.jsx | 493 ++++++++++++++++++ .../components/VoiceToTask/parseVoiceTask.js | 201 +++++++ .../components/VoiceToTask/useVoiceToTask.js | 217 ++++++++ .../components/VoiceToTask/voiceNormalizer.js | 129 +++++ 17 files changed, 1747 insertions(+), 17 deletions(-) create mode 100644 patches/@capacitor-community+speech-recognition+7.0.1.patch create mode 100644 src/service/VoiceInputService.js create mode 100644 src/views/components/VoiceToTask/VoicePanel.css create mode 100644 src/views/components/VoiceToTask/VoicePanel.jsx create mode 100644 src/views/components/VoiceToTask/parseVoiceTask.js create mode 100644 src/views/components/VoiceToTask/useVoiceToTask.js create mode 100644 src/views/components/VoiceToTask/voiceNormalizer.js diff --git a/android/app/capacitor.build.gradle b/android/app/capacitor.build.gradle index 739289a..932ca47 100644 --- a/android/app/capacitor.build.gradle +++ b/android/app/capacitor.build.gradle @@ -9,10 +9,12 @@ android { apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle" dependencies { + implementation project(':capacitor-community-speech-recognition') implementation project(':capacitor-community-sqlite') implementation project(':capacitor-app') implementation project(':capacitor-browser') implementation project(':capacitor-device') + implementation project(':capacitor-haptics') implementation project(':capacitor-local-llm') implementation project(':capacitor-local-notifications') implementation project(':capacitor-network') diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 1a34d79..2f15d14 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -117,4 +117,13 @@ + + + + + + + + + diff --git a/android/capacitor.settings.gradle b/android/capacitor.settings.gradle index a50983d..a1b0775 100644 --- a/android/capacitor.settings.gradle +++ b/android/capacitor.settings.gradle @@ -2,6 +2,9 @@ include ':capacitor-android' project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor') +include ':capacitor-community-speech-recognition' +project(':capacitor-community-speech-recognition').projectDir = new File('../node_modules/@capacitor-community/speech-recognition/android') + include ':capacitor-community-sqlite' project(':capacitor-community-sqlite').projectDir = new File('../node_modules/@capacitor-community/sqlite/android') @@ -14,6 +17,9 @@ project(':capacitor-browser').projectDir = new File('../node_modules/@capacitor/ include ':capacitor-device' project(':capacitor-device').projectDir = new File('../node_modules/@capacitor/device/android') +include ':capacitor-haptics' +project(':capacitor-haptics').projectDir = new File('../node_modules/@capacitor/haptics/android') + include ':capacitor-local-llm' project(':capacitor-local-llm').projectDir = new File('../node_modules/@capacitor/local-llm/android') diff --git a/ios/App/App/Info.plist b/ios/App/App/Info.plist index b960eaf..ada4f3e 100644 --- a/ios/App/App/Info.plist +++ b/ios/App/App/Info.plist @@ -28,6 +28,10 @@ This app needs access to camera to take photos to attach to task or use as profile photo NSPhotoLibraryUsageDescription This app needs access to photo library to select images to attach to task or use as profile photo + NSMicrophoneUsageDescription + This app uses the microphone to let you create tasks by speaking + NSSpeechRecognitionUsageDescription + This app uses on-device speech recognition to turn your voice into tasks UIBackgroundModes UILaunchStoryboardName diff --git a/ios/App/Podfile b/ios/App/Podfile index 30b894a..b4e2a32 100644 --- a/ios/App/Podfile +++ b/ios/App/Podfile @@ -11,10 +11,12 @@ install! 'cocoapods', :disable_input_output_paths => true def capacitor_pods pod 'Capacitor', :path => '../../node_modules/@capacitor/ios' pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios' + pod 'CapacitorCommunitySpeechRecognition', :path => '../../node_modules/@capacitor-community/speech-recognition' pod 'CapacitorCommunitySqlite', :path => '../../node_modules/@capacitor-community/sqlite' pod 'CapacitorApp', :path => '../../node_modules/@capacitor/app' pod 'CapacitorBrowser', :path => '../../node_modules/@capacitor/browser' pod 'CapacitorDevice', :path => '../../node_modules/@capacitor/device' + pod 'CapacitorHaptics', :path => '../../node_modules/@capacitor/haptics' pod 'CapacitorLocalLlm', :path => '../../node_modules/@capacitor/local-llm' pod 'CapacitorLocalNotifications', :path => '../../node_modules/@capacitor/local-notifications' pod 'CapacitorNetwork', :path => '../../node_modules/@capacitor/network' diff --git a/ios/App/Podfile.lock b/ios/App/Podfile.lock index 5f6f755..af25af6 100644 --- a/ios/App/Podfile.lock +++ b/ios/App/Podfile.lock @@ -18,6 +18,8 @@ PODS: - Capacitor - CapacitorBrowser (8.0.3): - Capacitor + - CapacitorCommunitySpeechRecognition (7.0.1): + - Capacitor - CapacitorCommunitySqlite (8.1.0): - Capacitor - SQLCipher @@ -25,6 +27,8 @@ PODS: - CapacitorCordova (8.4.1) - CapacitorDevice (8.0.2): - Capacitor + - CapacitorHaptics (8.0.2): + - Capacitor - CapacitorLocalLlm (1.0.0): - Capacitor - CapacitorLocalNotifications (8.2.0): @@ -149,9 +153,11 @@ DEPENDENCIES: - "Capacitor (from `../../node_modules/@capacitor/ios`)" - "CapacitorApp (from `../../node_modules/@capacitor/app`)" - "CapacitorBrowser (from `../../node_modules/@capacitor/browser`)" + - "CapacitorCommunitySpeechRecognition (from `../../node_modules/@capacitor-community/speech-recognition`)" - "CapacitorCommunitySqlite (from `../../node_modules/@capacitor-community/sqlite`)" - "CapacitorCordova (from `../../node_modules/@capacitor/ios`)" - "CapacitorDevice (from `../../node_modules/@capacitor/device`)" + - "CapacitorHaptics (from `../../node_modules/@capacitor/haptics`)" - "CapacitorLocalLlm (from `../../node_modules/@capacitor/local-llm`)" - "CapacitorLocalNotifications (from `../../node_modules/@capacitor/local-notifications`)" - "CapacitorNetwork (from `../../node_modules/@capacitor/network`)" @@ -203,12 +209,16 @@ EXTERNAL SOURCES: :path: "../../node_modules/@capacitor/app" CapacitorBrowser: :path: "../../node_modules/@capacitor/browser" + CapacitorCommunitySpeechRecognition: + :path: "../../node_modules/@capacitor-community/speech-recognition" CapacitorCommunitySqlite: :path: "../../node_modules/@capacitor-community/sqlite" CapacitorCordova: :path: "../../node_modules/@capacitor/ios" CapacitorDevice: :path: "../../node_modules/@capacitor/device" + CapacitorHaptics: + :path: "../../node_modules/@capacitor/haptics" CapacitorLocalLlm: :path: "../../node_modules/@capacitor/local-llm" CapacitorLocalNotifications: @@ -243,9 +253,11 @@ SPEC CHECKSUMS: Capacitor: 35242afe195b1e53c58ca1b827d1b444c5e6602b CapacitorApp: 449ffe26375e96f8aaaee625ac6e01e5c57c8650 CapacitorBrowser: c987c73d09d8bd3b5ec13f06338b1e14d5d2be69 + CapacitorCommunitySpeechRecognition: 3e03566c44c2bb3b52391a33d4518b1adbdeb38f CapacitorCommunitySqlite: eac6acfb852f46e7988fc59604d7f900498d354e CapacitorCordova: eebe6bcf807b1b06f3f48237650f96bbcd0eef09 CapacitorDevice: 14cba6f88d1c3074cbf825fea977c8c526453ff8 + CapacitorHaptics: 296f771ecd89c7a1bd92a7b6826a7d268e2e70f5 CapacitorLocalLlm: a05516151a02923a9e7dae9949d3817e85e321f0 CapacitorLocalNotifications: 2615aa008f608b95d3921a778ee1988abf1e6148 CapacitorNetwork: 8812ce60d11fb63d8f2e4ba51a49b2e59892ebe2 @@ -283,6 +295,6 @@ SPEC CHECKSUMS: SQLCipher: eb79c64049cb002b4e9fcb30edb7979bf4706dfc ZIPFoundation: dfd3d681c4053ff7e2f7350bc4e53b5dba3f5351 -PODFILE CHECKSUM: 21b805bdbbb6ac4b8a3527ee8ef346ba0a55ef65 +PODFILE CHECKSUM: 1099083fe561f8852fcef1bfcff4051f45db9770 COCOAPODS: 1.16.2 diff --git a/package-lock.json b/package-lock.json index ee5471e..f496011 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,22 @@ { "name": "donetick", - "version": "1.2.15", + "version": "1.2.16", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "donetick", - "version": "1.2.15", + "version": "1.2.16", + "hasInstallScript": true, "dependencies": { + "@capacitor-community/speech-recognition": "^7.0.1", "@capacitor-community/sqlite": "^8.0.0", "@capacitor/android": "^8.0.0", "@capacitor/app": "^8.0.0", "@capacitor/browser": "^8.0.0", "@capacitor/core": "^8.0.0", "@capacitor/device": "^8.0.0", + "@capacitor/haptics": "^8.0.2", "@capacitor/ios": "^8.0.0", "@capacitor/local-llm": "^1.0.0", "@capacitor/local-notifications": "^8.0.0", @@ -94,6 +97,7 @@ "eslint-plugin-sort-keys-fix": "^1.1.2", "eslint-plugin-tailwindcss": "^3.13.1", "husky": "^8.0.3", + "patch-package": "^8.0.1", "postcss": "^8.4.32", "prettier": "^3.1.1", "prettier-plugin-tailwindcss": "^0.5.10", @@ -1464,6 +1468,15 @@ "devOptional": true, "license": "MIT" }, + "node_modules/@capacitor-community/speech-recognition": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@capacitor-community/speech-recognition/-/speech-recognition-7.0.1.tgz", + "integrity": "sha512-ykpBZziR575X0eURO5vXaD9gVrXXC/7Ra2qql/2KP6/jxWOqAFuw4eKjSPjwdAgGE6a/Z+v8FJm4SVh57MVwDA==", + "license": "MIT", + "peerDependencies": { + "@capacitor/core": ">=7.0.0" + } + }, "node_modules/@capacitor-community/sqlite": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/@capacitor-community/sqlite/-/sqlite-8.1.0.tgz", @@ -2057,6 +2070,15 @@ "@capacitor/core": ">=8.0.0" } }, + "node_modules/@capacitor/haptics": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@capacitor/haptics/-/haptics-8.0.2.tgz", + "integrity": "sha512-c2hZzRR5Fk1tbTvhG1jhh2XBAf3EhnIerMIb2sl7Mt41Gxx1fhBJFDa0/BI1IbY4loVepyyuqNC9820/GZuoWQ==", + "license": "MIT", + "peerDependencies": { + "@capacitor/core": ">=8.0.0" + } + }, "node_modules/@capacitor/ios": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@capacitor/ios/-/ios-8.4.1.tgz", @@ -5518,6 +5540,13 @@ "node": ">=10.0.0" } }, + "node_modules/@yarnpkg/lockfile": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", + "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/acorn": { "version": "8.15.0", "license": "MIT", @@ -6767,6 +6796,22 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/classlist-polyfill": { "version": "1.2.0", "license": "Unlicense" @@ -8677,6 +8722,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/find-yarn-workspace-root": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz", + "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "micromatch": "^4.0.2" + } + }, "node_modules/flat-cache": { "version": "3.2.0", "dev": true, @@ -10135,11 +10190,38 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stable-stringify": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz", + "integrity": "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "isarray": "^2.0.5", + "jsonify": "^0.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true, "license": "MIT" }, + "node_modules/json-stable-stringify/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "dev": true, + "license": "MIT" + }, "node_modules/json-stringify-safe": { "version": "5.0.1", "dev": true, @@ -10165,6 +10247,16 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz", + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==", + "dev": true, + "license": "Public Domain", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/jsonparse": { "version": "1.3.1", "dev": true, @@ -10235,6 +10327,16 @@ "node": ">=0.10.0" } }, + "node_modules/klaw-sync": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", + "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.11" + } + }, "node_modules/kleur": { "version": "4.1.5", "dev": true, @@ -11322,6 +11424,79 @@ "cross-spawn": "^7.0.3" } }, + "node_modules/patch-package": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.1.tgz", + "integrity": "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@yarnpkg/lockfile": "^1.1.0", + "chalk": "^4.1.2", + "ci-info": "^3.7.0", + "cross-spawn": "^7.0.3", + "find-yarn-workspace-root": "^2.0.0", + "fs-extra": "^10.0.0", + "json-stable-stringify": "^1.0.2", + "klaw-sync": "^6.0.0", + "minimist": "^1.2.6", + "open": "^7.4.2", + "semver": "^7.5.3", + "slash": "^2.0.0", + "tmp": "^0.2.4", + "yaml": "^2.2.2" + }, + "bin": { + "patch-package": "index.js" + }, + "engines": { + "node": ">=14", + "npm": ">5" + } + }, + "node_modules/patch-package/node_modules/open": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", + "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/patch-package/node_modules/slash": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/patch-package/node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/path-exists": { "version": "4.0.0", "license": "MIT", diff --git a/package.json b/package.json index 11c19fd..4a3d849 100644 --- a/package.json +++ b/package.json @@ -35,15 +35,18 @@ "bump": "node bump-version.js patch", "bump:minor": "node bump-version.js minor", "bump:major": "node bump-version.js major", - "bump:patch": "node bump-version.js patch" + "bump:patch": "node bump-version.js patch", + "postinstall": "patch-package" }, "dependencies": { + "@capacitor-community/speech-recognition": "^7.0.1", "@capacitor-community/sqlite": "^8.0.0", "@capacitor/android": "^8.0.0", "@capacitor/app": "^8.0.0", "@capacitor/browser": "^8.0.0", "@capacitor/core": "^8.0.0", "@capacitor/device": "^8.0.0", + "@capacitor/haptics": "^8.0.2", "@capacitor/ios": "^8.0.0", "@capacitor/local-llm": "^1.0.0", "@capacitor/local-notifications": "^8.0.0", @@ -124,6 +127,7 @@ "eslint-plugin-sort-keys-fix": "^1.1.2", "eslint-plugin-tailwindcss": "^3.13.1", "husky": "^8.0.3", + "patch-package": "^8.0.1", "postcss": "^8.4.32", "prettier": "^3.1.1", "prettier-plugin-tailwindcss": "^0.5.10", diff --git a/patches/@capacitor-community+speech-recognition+7.0.1.patch b/patches/@capacitor-community+speech-recognition+7.0.1.patch new file mode 100644 index 0000000..851fc64 --- /dev/null +++ b/patches/@capacitor-community+speech-recognition+7.0.1.patch @@ -0,0 +1,26 @@ +diff --git a/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java b/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java +index a99ea69..2bea32e 100644 +--- a/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java ++++ b/node_modules/@capacitor-community/speech-recognition/android/src/main/java/com/getcapacitor/community/speechrecognition/SpeechRecognition.java +@@ -168,6 +168,7 @@ public class SpeechRecognition extends Plugin implements Constants { + intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, bridge.getActivity().getPackageName()); + intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, partialResults); + intent.putExtra("android.speech.extra.DICTATION_MODE", partialResults); ++ intent.putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true); + + if (prompt != null) { + intent.putExtra(RecognizerIntent.EXTRA_PROMPT, prompt); +diff --git a/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift b/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift +index 5d1b35b..810277d 100644 +--- a/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift ++++ b/node_modules/@capacitor-community/speech-recognition/ios/Plugin/Plugin.swift +@@ -79,6 +79,9 @@ public class SpeechRecognition: CAPPlugin { + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = partialResults ++ if #available(iOS 13, *), self.speechRecognizer?.supportsOnDeviceRecognition == true { ++ self.recognitionRequest?.requiresOnDeviceRecognition = true ++ } + + let inputNode: AVAudioInputNode = self.audioEngine!.inputNode + let format: AVAudioFormat = inputNode.outputFormat(forBus: 0) diff --git a/src/service/VoiceInputService.js b/src/service/VoiceInputService.js new file mode 100644 index 0000000..f752b17 --- /dev/null +++ b/src/service/VoiceInputService.js @@ -0,0 +1,256 @@ +import { Capacitor } from '@capacitor/core' + +// Platform-abstracted speech-to-text for voice task capture. +// Native: @capacitor-community/speech-recognition — uses the OS recognizer +// (on-device where the platform supports it, e.g. iOS dictation models). +// Web: Web Speech API (Chrome/Safari) — mainly for development. +// +// Callbacks: +// onPartial(text) — live transcript of the utterance in progress +// onSegment(text) — a finalized utterance (silence/pause boundary) +// onStateChange(bool) — listening started/stopped +// onError(code) — 'denied' | 'error' +// +// Neither OS gives unlimited continuous listening: Android's recognizer ends on +// silence and iOS sessions have a practical duration limit. This service runs a +// restart loop — each recognizer stop commits the buffered utterance as a +// segment and immediately starts a new session while active. The utterance +// boundary doubles as the task boundary. + +const SILENCE_COMMIT_MS = 2200 +const RESTART_DELAY_MS = 250 + +const START_OPTIONS = { + language: 'en-US', + maxResults: 1, + partialResults: true, + popup: false, +} + +class VoiceInputService { + constructor() { + this._active = false + this._callbacks = null + this._partial = '' + this._lastSpeechAt = 0 + this._silenceTimer = null + this._restarting = false + this._webRecognition = null + } + + get isNative() { + return Capacitor.isNativePlatform() + } + + async isSupported() { + if (this.isNative) { + try { + const { SpeechRecognition } = await import( + '@capacitor-community/speech-recognition' + ) + const { available } = await SpeechRecognition.available() + return !!available + } catch { + return false + } + } + return ( + typeof window !== 'undefined' && + !!(window.SpeechRecognition || window.webkitSpeechRecognition) + ) + } + + async requestPermission() { + if (!this.isNative) { + // Web prompts for the microphone on first start() + return 'granted' + } + try { + const { SpeechRecognition } = await import( + '@capacitor-community/speech-recognition' + ) + const current = await SpeechRecognition.checkPermissions() + if (current.speechRecognition === 'granted') return 'granted' + const res = await SpeechRecognition.requestPermissions() + return res.speechRecognition === 'granted' ? 'granted' : 'denied' + } catch { + return 'denied' + } + } + + async start(callbacks) { + if (this._active) return + this._callbacks = callbacks + this._active = true + this._partial = '' + this._lastSpeechAt = Date.now() + + if (this.isNative) { + await this._startNative() + // Web finalizes utterances itself via isFinal results; only the native + // path needs a silence watchdog to force utterance boundaries. + this._silenceTimer = setInterval(() => this._checkSilence(), 500) + } else { + this._startWeb() + } + this._callbacks?.onStateChange?.(true) + } + + async stop() { + if (!this._active) return + this._active = false + if (this._silenceTimer) { + clearInterval(this._silenceTimer) + this._silenceTimer = null + } + if (this.isNative) { + try { + const { SpeechRecognition } = await import( + '@capacitor-community/speech-recognition' + ) + await SpeechRecognition.stop() + await SpeechRecognition.removeAllListeners() + } catch { + // recognizer may already be stopped + } + } else if (this._webRecognition) { + const rec = this._webRecognition + this._webRecognition = null + try { + rec.stop() + } catch { + // already stopped + } + } + this._commitPartial() + this._callbacks?.onStateChange?.(false) + } + + _commitPartial() { + const text = this._partial.trim() + this._partial = '' + this._callbacks?.onPartial?.('') + if (text) this._callbacks?.onSegment?.(text) + } + + _checkSilence() { + if (!this._active || this._restarting) return + if ( + this._partial.trim() && + Date.now() - this._lastSpeechAt > SILENCE_COMMIT_MS + ) { + // A pause means the utterance (= task) is complete: cycle the recognizer + // so the buffer commits and a fresh session begins. + this._restartNative() + } + } + + async _startNative() { + const { SpeechRecognition } = await import( + '@capacitor-community/speech-recognition' + ) + await SpeechRecognition.removeAllListeners() + + await SpeechRecognition.addListener('partialResults', ({ matches }) => { + const text = matches?.[0] || '' + if (!text) return + this._partial = text + this._lastSpeechAt = Date.now() + this._callbacks?.onPartial?.(text) + }) + + await SpeechRecognition.addListener('listeningState', ({ status }) => { + if (status === 'stopped' && this._active && !this._restarting) { + // OS ended the session on its own (silence on Android, session limit + // on iOS) — commit and start over. + this._restartNative() + } + }) + + // With partialResults the transcript arrives via listeners; the promise's + // resolution/rejection timing differs per platform, so don't rely on it. + SpeechRecognition.start(START_OPTIONS).catch(() => { + if (this._active && !this._restarting) { + this._restartNative() + } + }) + } + + async _restartNative() { + if (this._restarting) return + this._restarting = true + try { + const { SpeechRecognition } = await import( + '@capacitor-community/speech-recognition' + ) + try { + await SpeechRecognition.stop() + } catch { + // already stopped + } + this._commitPartial() + // Let the OS recognizer tear down before starting a new session + await new Promise(r => setTimeout(r, RESTART_DELAY_MS)) + if (this._active) { + SpeechRecognition.start(START_OPTIONS).catch(() => {}) + this._lastSpeechAt = Date.now() + } + } finally { + this._restarting = false + } + } + + _startWeb() { + const SR = window.SpeechRecognition || window.webkitSpeechRecognition + const rec = new SR() + rec.continuous = true + rec.interimResults = true + rec.lang = 'en-US' + + rec.onresult = event => { + let interim = '' + for (let i = event.resultIndex; i < event.results.length; i++) { + const res = event.results[i] + if (res.isFinal) { + this._partial = '' + this._callbacks?.onPartial?.('') + const text = res[0].transcript.trim() + if (text) this._callbacks?.onSegment?.(text) + } else { + interim += res[0].transcript + } + } + if (interim) { + this._partial = interim + this._lastSpeechAt = Date.now() + this._callbacks?.onPartial?.(interim) + } + } + + rec.onend = () => { + if (this._active && this._webRecognition === rec) { + try { + rec.start() + } catch { + // restart can race with teardown + } + } + } + + rec.onerror = e => { + if (e.error === 'not-allowed' || e.error === 'service-not-allowed') { + this._callbacks?.onError?.('denied') + this.stop() + } + } + + this._webRecognition = rec + try { + rec.start() + } catch { + this._callbacks?.onError?.('error') + } + } +} + +export const voiceInputService = new VoiceInputService() diff --git a/src/views/components/AddTaskModal.jsx b/src/views/components/AddTaskModal.jsx index 30ed6f5..bda067c 100644 --- a/src/views/components/AddTaskModal.jsx +++ b/src/views/components/AddTaskModal.jsx @@ -24,6 +24,7 @@ import SmartTaskTitleInput from './SmartTaskTitleInput' import KeyboardShortcutHint from '../../components/common/KeyboardShortcutHint' import { useDocumentScanner } from '../../hooks/useDocumentScanner' import { localAIService } from '../../service/LocalAIService' +import { voiceInputService } from '../../service/VoiceInputService' import { TASK_COLOR } from '../../utils/Colors' import AdvancedOptionsSection, { AdvancedOptionsTrigger, @@ -39,6 +40,8 @@ import RepeatPickerField from './RepeatPickerField' import RichTextEditor from './RichTextEditor' import ScanPanel from './ScanToTask/ScanPanel' import SubTasks from './SubTask' +import { buildChorePayload } from './VoiceToTask/parseVoiceTask' +import VoicePanel from './VoiceToTask/VoicePanel' const getDefaultNotification = () => { const storedDefault = localStorage.getItem('defaultNotificationTemplate') if (storedDefault) { @@ -121,10 +124,13 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => { const [scanAutoCapture, setScanAutoCapture] = useState(false) const [pendingPhotoUrl, setPendingPhotoUrl] = useState(null) const [llmAvailable, setLlmAvailable] = useState(false) + const [showVoice, setShowVoice] = useState(false) + const [voiceAvailable, setVoiceAvailable] = useState(false) const { isNativeScanner } = useDocumentScanner() useEffect(() => { localAIService.isAvailable().then(setLlmAvailable) + voiceInputService.isSupported().then(setVoiceAvailable) }, []) // Priority colors @@ -589,9 +595,46 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => { } } + // Single voice-captured task: land it in the smart input so the user + // reviews it with the normal pickers before creating. + const handleVoiceSingle = text => { + setShowVoice(false) + processText(text) + } + + // Multiple voice-captured tasks: they were reviewed as cards in the panel, + // so create them all directly. + const handleVoiceCreateMany = async parsedTasks => { + const notificationTemplates = getDefaultNotification() + for (const parsed of parsedTasks) { + const chore = buildChorePayload(parsed, { + userProfile, + projectId, + notificationTemplates, + }) + try { + const result = await createChoreMutation.mutateAsync(chore) + if (result?._pendingCreate) { + onChoreUpdate(result) + } else { + onChoreUpdate({ + ...chore, + ...result, + id: result?.id, + nextDueDate: chore.dueDate, + }) + } + } catch (error) { + console.error('Error creating voice task:', error) + } + } + handleCloseModal(false) + } + const handleCloseModal = forceRefetch => { onClose(forceRefetch) setShowScan(false) + setShowVoice(false) setTaskText('') setTaskTitle('') setDueDate(null) @@ -761,7 +804,7 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => { } > - {!showScan && ( + {!showScan && !showVoice && ( <> { } : undefined } + onVoiceClick={ + voiceAvailable ? () => setShowVoice(true) : undefined + } placeholder='Type your task...' onChange={text => { setTaskText(text) @@ -1097,6 +1143,18 @@ const TaskInput = ({ onChoreUpdate, isModalOpen, onClose }) => { )} + {showVoice && ( + setShowVoice(false)} + onUseSingle={handleVoiceSingle} + onCreateMany={handleVoiceCreateMany} + /> + )} + {showScan && ( { const { mode, setMode } = useColorScheme() const titleInputRef = useRef(null) @@ -200,13 +201,14 @@ const SmartTaskTitleInput = ({ e.target.value = '' } - const showNativeButtons = isNativeScanner && !value - const MIC_BUTTON_WIDTH = - showNativeButtons && onPhotoSelected && onScanClick - ? '5rem' - : showNativeButtons - ? '2.5rem' - : '0rem' + const showPhotoButtons = isNativeScanner && !value + const showVoiceButton = !!onVoiceClick && !value + const visibleButtonCount = + (showPhotoButtons && onPhotoSelected ? 1 : 0) + + (showPhotoButtons && onScanClick ? 1 : 0) + + (showVoiceButton ? 1 : 0) + const showActionButtons = visibleButtonCount > 0 + const ACTION_BUTTONS_WIDTH = `${visibleButtonCount * 2.5}rem` return (
@@ -223,7 +225,7 @@ const SmartTaskTitleInput = ({ position: 'absolute', top: 0, left: 0, - width: `calc(100% - ${MIC_BUTTON_WIDTH})`, + width: `calc(100% - ${ACTION_BUTTONS_WIDTH})`, height: '100%', zIndex: 1, resize: 'none', @@ -274,7 +276,7 @@ const SmartTaskTitleInput = ({ {/* Zero-width space to maintain consistent height */} ​
- {showNativeButtons && ( + {showActionButtons && ( - {onPhotoSelected && ( + {showPhotoButtons && onPhotoSelected && ( <> )} - {onScanClick && ( + {showPhotoButtons && onScanClick && ( )} + {showVoiceButton && ( + + + + + + )} )} diff --git a/src/views/components/VoiceToTask/VoicePanel.css b/src/views/components/VoiceToTask/VoicePanel.css new file mode 100644 index 0000000..f8648a7 --- /dev/null +++ b/src/views/components/VoiceToTask/VoicePanel.css @@ -0,0 +1,121 @@ +.voice-mic-btn { + position: relative; + width: 72px; + height: 72px; + border-radius: 50%; + border: none; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + background: var(--joy-palette-primary-solidBg, #0b6bcb); + color: #fff; + transition: + transform 0.15s ease, + background 0.2s ease, + box-shadow 0.2s ease; + touch-action: none; + user-select: none; + -webkit-user-select: none; + -webkit-tap-highlight-color: transparent; +} + +.voice-mic-btn:active { + transform: scale(0.94); +} + +.voice-mic-btn.listening { + background: var(--joy-palette-danger-solidBg, #c41c1c); + box-shadow: 0 4px 18px rgba(196, 28, 28, 0.35); +} + +.voice-pulse-ring { + position: absolute; + inset: 0; + border-radius: 50%; + pointer-events: none; + opacity: 0; +} + +.voice-mic-btn.listening .voice-pulse-ring { + opacity: 1; + animation: voice-pulse 1.8s ease-out infinite; +} + +.voice-mic-btn.listening .voice-pulse-ring:nth-child(2) { + animation-delay: 0.6s; +} + +@keyframes voice-pulse { + 0% { + box-shadow: 0 0 0 0 rgba(196, 28, 28, 0.4); + } + 100% { + box-shadow: 0 0 0 26px rgba(196, 28, 28, 0); + } +} + +/* Faux equalizer shown while listening */ +.voice-eq { + display: flex; + gap: 3px; + align-items: center; + height: 28px; +} + +.voice-eq span { + width: 4px; + border-radius: 2px; + background: var(--joy-palette-danger-solidBg, #c41c1c); + animation: voice-eq-wave 1.1s ease-in-out infinite; +} + +.voice-eq span:nth-child(1) { + animation-delay: 0s; +} +.voice-eq span:nth-child(2) { + animation-delay: 0.18s; +} +.voice-eq span:nth-child(3) { + animation-delay: 0.32s; +} +.voice-eq span:nth-child(4) { + animation-delay: 0.12s; +} +.voice-eq span:nth-child(5) { + animation-delay: 0.26s; +} + +@keyframes voice-eq-wave { + 0%, + 100% { + height: 6px; + } + 50% { + height: 24px; + } +} + +/* Committed task cards slide in as segments are captured */ +.voice-task-card { + animation: voice-card-in 0.25s ease-out; +} + +@keyframes voice-card-in { + from { + opacity: 0; + transform: translateY(6px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +@media (prefers-reduced-motion: reduce) { + .voice-mic-btn.listening .voice-pulse-ring, + .voice-eq span, + .voice-task-card { + animation: none; + } +} diff --git a/src/views/components/VoiceToTask/VoicePanel.jsx b/src/views/components/VoiceToTask/VoicePanel.jsx new file mode 100644 index 0000000..f271fa6 --- /dev/null +++ b/src/views/components/VoiceToTask/VoicePanel.jsx @@ -0,0 +1,493 @@ +import { + CalendarMonth, + Close, + Flag, + GraphicEq, + Lock, + Mic, + Person, + Repeat, + Sell, + Toll, + WarningAmber, +} from '@mui/icons-material' +import { Box, Button, Chip, IconButton, Input, Typography } from '@mui/joy' +import moment from 'moment' +import { useMemo, useState } from 'react' +import { parseVoiceTask } from './parseVoiceTask' +import { useVoiceToTask } from './useVoiceToTask' +import './VoicePanel.css' + +const HIGHLIGHT_CLASS = { + repeat: 'highlight-repeat', + priority: 'highlight-priority', + points: 'highlight-points', + assignee: 'highlight-assignee', + label: 'highlight-label', + dueDate: 'highlight-date', +} + +const renderTranscript = (text, highlights) => { + const parts = [] + let lastIndex = 0 + for (const h of highlights) { + if (h.start > lastIndex) parts.push(text.substring(lastIndex, h.start)) + parts.push( + + {text.substring(h.start, h.end)} + , + ) + lastIndex = h.end + } + if (lastIndex < text.length) parts.push(text.substring(lastIndex)) + return parts +} + +const formatDue = dueDate => { + const m = moment(dueDate) + return m.format('HH:mm') === '23:59' + ? m.format('MMM D') + : m.format('MMM D, h:mm A') +} + +const buildChips = (parsed, { members, currentUserId }) => { + const chips = [] + if (parsed.dueDate) { + chips.push({ + key: 'due', + color: 'warning', + icon: , + label: formatDue(parsed.dueDate), + }) + } + if (parsed.frequencyName) { + chips.push({ + key: 'repeat', + color: 'success', + icon: , + label: parsed.frequencyName, + }) + } + if (parsed.priority > 0) { + chips.push({ + key: 'priority', + color: 'danger', + icon: , + label: `P${parsed.priority}`, + }) + } + if (parsed.points != null) { + chips.push({ + key: 'points', + color: 'primary', + icon: , + label: `${parsed.points} pts`, + }) + } + parsed.labelNames.forEach(name => { + chips.push({ + key: `label-${name}`, + color: 'primary', + icon: , + label: name, + }) + }) + if (parsed.isAnyone) { + chips.push({ + key: 'assignee', + color: 'neutral', + icon: , + label: 'Anyone', + }) + } else if ( + parsed.assignees.length > 0 && + parsed.assignees[0].userId !== currentUserId + ) { + const member = members.find(m => m.userId === parsed.assignees[0].userId) + if (member) { + chips.push({ + key: 'assignee', + color: 'neutral', + icon: , + label: member.displayName, + }) + } + } + return chips +} + +const TaskPreviewCard = ({ segment, parseCtx, onRemove, onUpdate }) => { + const [editing, setEditing] = useState(false) + const [draft, setDraft] = useState(segment.text) + + const parsed = useMemo( + () => parseVoiceTask(segment.text, parseCtx), + [segment.text, parseCtx], + ) + const chips = useMemo(() => buildChips(parsed, parseCtx), [parsed, parseCtx]) + + const commitEdit = () => { + setEditing(false) + if (draft.trim() !== segment.text) onUpdate(draft) + } + + return ( + + + {editing ? ( + setDraft(e.target.value)} + onKeyDown={e => { + if (e.key === 'Enter') commitEdit() + if (e.key === 'Escape') { + setDraft(segment.text) + setEditing(false) + } + }} + onBlur={commitEdit} + sx={{ flex: 1 }} + /> + ) : ( + { + setDraft(segment.text) + setEditing(true) + }} + > + {parsed.title || segment.text} + + )} + + + + + {chips.length > 0 && ( + + {chips.map(chip => ( + + {chip.label} + + ))} + + )} + + ) +} + +/** + * Inline voice-to-task panel. Mounts inside AddTaskModal — no second modal. + * + * Hold the mic to speak, or tap once for hands-free. Pauses and spoken + * separators ("also") split the transcript into task cards. A single captured + * task lands in the smart input for review; multiple tasks are created + * directly from the review list. + */ +const VoicePanel = ({ + open, + userLabels = [], + members = [], + userProfile, + onClose, + onUseSingle, + onCreateMany, +}) => { + const { + phase, + isLocked, + partialText, + segments, + micPressDown, + micPressUp, + startListening, + removeSegment, + updateSegment, + reset, + isNative, + } = useVoiceToTask({ members }) + const [creating, setCreating] = useState(false) + + const parseCtx = useMemo( + () => ({ userLabels, members, currentUserId: userProfile?.id }), + [userLabels, members, userProfile?.id], + ) + + const partialParsed = useMemo( + () => (partialText ? parseVoiceTask(partialText, parseCtx) : null), + [partialText, parseCtx], + ) + + if (!open) return null + + const isListening = phase === 'listening' + const showActions = segments.length > 0 && !isListening && !creating + + const handleCancel = () => { + reset() + onClose() + } + + const handleCreateAll = async () => { + setCreating(true) + try { + await onCreateMany(segments.map(s => parseVoiceTask(s.text, parseCtx))) + } finally { + setCreating(false) + } + } + + const micCaption = isListening + ? isLocked + ? 'Listening — tap to stop' + : 'Release to finish · quick tap locks hands-free' + : segments.length > 0 + ? 'Hold to add another task' + : 'Hold to speak · quick tap for hands-free' + + return ( + + {/* ── Header ── */} + + + Speak your tasks + {isNative && ( + } + sx={{ ml: 'auto' }} + > + On-device + + )} + + + {/* ── Permission denied ── */} + {phase === 'denied' && ( + + + + + Microphone access is needed for voice capture. Enable it in your + device settings and try again. + + + + + )} + + {/* ── Captured task cards ── */} + {segments.length > 0 && ( + + {segments.map(segment => ( + removeSegment(segment.id)} + onUpdate={text => updateSegment(segment.id, text)} + /> + ))} + + )} + + {/* ── Live transcript ── */} + {isListening && ( + + + {partialText ? ( + + {renderTranscript(partialText, partialParsed?.highlights || [])} + + ) : ( + + Listening… + + )} + + + )} + + {/* ── Mic stage ── */} + {phase !== 'denied' && ( + +
+ + + + + +
+ + + {micCaption} + + + Pause or say “also” between tasks · say + “scratch that” to remove the last one + + + )} + + {/* ── Footer ── */} + + + + {creating && ( + + )} + {showActions && + (segments.length === 1 ? ( + + ) : ( + + ))} + + + + ) +} + +export default VoicePanel diff --git a/src/views/components/VoiceToTask/parseVoiceTask.js b/src/views/components/VoiceToTask/parseVoiceTask.js new file mode 100644 index 0000000..ad46871 --- /dev/null +++ b/src/views/components/VoiceToTask/parseVoiceTask.js @@ -0,0 +1,201 @@ +import * as chrono from 'chrono-node' +import moment from 'moment' +import { isPlusAccount } from '../../../utils/Helpers' +import { generateUUID } from '../../../utils/UUID' +import { + parseAssignees, + parseDueDate, + parseLabels, + parsePoints, + parsePriority, + parseRepeatV2, +} from '../CustomParsers' + +// Pure equivalent of AddTaskModal.processText — parses one sentence into a +// structured task (no state setters), preserving the same parser order and +// sequential-cleanup behavior so voice and typed input stay consistent. + +const mapMembersForParsing = members => + members.map(member => ({ + userId: member.userId, + username: + member.username || member.displayName?.toLowerCase().replace(/\s+/g, ''), + displayName: member.displayName, + name: member.displayName, + id: member.userId, + })) + +// Merge overlapping highlight ranges, higher parser priority wins — same +// resolution rules as AddTaskModal.renderHighlightedSentence. +const resolveHighlights = ({ + repeat, + priority, + points, + assignees, + labels, + dueDate, +}) => { + const all = [] + repeat?.forEach(h => all.push({ ...h, type: 'repeat', rank: 60 })) + priority?.forEach(h => all.push({ ...h, type: 'priority', rank: 50 })) + points?.forEach(h => all.push({ ...h, type: 'points', rank: 45 })) + assignees?.forEach(h => all.push({ ...h, type: 'assignee', rank: 40 })) + labels?.forEach(h => all.push({ ...h, type: 'label', rank: 30 })) + if (dueDate) all.push({ ...dueDate, type: 'dueDate', rank: 20 }) + + all.sort((a, b) => a.start - b.start) + const resolved = [] + for (const current of all) { + const previous = resolved[resolved.length - 1] + if (previous && current.start < previous.end) { + if (current.rank > previous.rank) { + resolved.pop() + resolved.push(current) + } + } else { + resolved.push(current) + } + } + return resolved +} + +export const parseVoiceTask = ( + sentence, + { userLabels = [], members = [], currentUserId = null } = {}, +) => { + const assigneesForParsing = mapMembersForParsing(members) + + const priority = parsePriority(sentence) + const points = parsePoints(sentence) + const labels = parseLabels(sentence, userLabels) + const assigneesResult = parseAssignees(sentence, assigneesForParsing) + const repeat = parseRepeatV2(sentence) + const dueDateParsed = parseDueDate(sentence, chrono) + + // Sequential cleanup — identical chain to AddTaskModal.processText + let cleaned = sentence + if (priority.result) cleaned = priority.cleanedSentence + if (points.result) { + const reparse = parsePoints(cleaned) + if (reparse.result) cleaned = reparse.cleanedSentence + } + if (labels.result) { + const reparse = parseLabels(cleaned, userLabels) + if (reparse.result) cleaned = reparse.cleanedSentence + } + if (assigneesResult.result) { + const reparse = parseAssignees(cleaned, assigneesForParsing) + if (reparse.result) cleaned = reparse.cleanedSentence + } + if (repeat.result) { + const reparse = parseRepeatV2(cleaned) + if (reparse.result) cleaned = reparse.cleanedSentence + } + if (dueDateParsed.result) { + const reparse = parseDueDate(cleaned, chrono) + if (reparse.result) cleaned = reparse.cleanedSentence + } + + let dueDate = null + if (dueDateParsed.result) { + dueDate = moment(dueDateParsed.result).format('YYYY-MM-DDTHH:mm:ss') + } else if (repeat.dueDate) { + dueDate = moment(repeat.dueDate).format('YYYY-MM-DDTHH:mm:ss') + } + + let assignees = [] + const isAnyone = !!assigneesResult.isAnyone + if (!isAnyone) { + if (assigneesResult.result?.length > 0) { + assignees = assigneesResult.result.map(a => ({ userId: a.userId })) + } else if (currentUserId) { + assignees = [{ userId: currentUserId }] + } + } + + const labelIds = (labels.result || []) + .filter(label => label.id) + .map(label => label.id) + + return { + raw: sentence, + title: cleaned.replace(/\s+/g, ' ').trim(), + priority: priority.result ? parseInt(priority.result, 10) : 0, + points: points.result ?? null, + labelIds, + labelNames: (labels.result || []).map(label => label.name), + assignees, + isAnyone, + frequency: repeat.result, + frequencyName: repeat.name, + dueDate, + highlights: resolveHighlights({ + repeat: repeat.highlight, + priority: priority.highlight, + points: points.highlight, + assignees: assigneesResult.highlight, + labels: labels.highlight, + dueDate: dueDateParsed.result ? dueDateParsed.highlight[0] : null, + }), + } +} + +// Builds the same chore payload shape AddTaskModal.createChore submits. +export const buildChorePayload = ( + parsed, + { userProfile, projectId, notificationTemplates }, +) => { + let finalAssignees = parsed.assignees + let finalAssignedTo = null + let finalAssignStrategy = 'keep_last_assigned' + + if (parsed.isAnyone) { + finalAssignees = [] + finalAssignStrategy = 'no_assignee' + } else if (finalAssignees.length === 0) { + finalAssignees = [{ userId: userProfile?.id }] + finalAssignedTo = userProfile?.id + } else { + finalAssignedTo = finalAssignees[0].userId + } + + const chore = { + name: parsed.title, + description: null, + assignees: finalAssignees, + dueDate: parsed.dueDate ? new Date(parsed.dueDate).toISOString() : null, + assignedTo: finalAssignedTo, + assignStrategy: finalAssignStrategy, + isRolling: false, + labelsV2: parsed.labelIds, + priority: parsed.priority || 0, + points: parsed.points ?? null, + deadlineOffset: null, + completionWindow: null, + requireApproval: false, + isPrivate: false, + status: 0, + frequencyType: 'once', + frequencyMetadata: {}, + notificationMetadata: {}, + subTasks: null, + projectId: projectId === 'default' ? null : projectId, + draftId: generateUUID(), + } + + if (parsed.frequency) { + chore.frequencyType = parsed.frequency.frequencyType + chore.frequencyMetadata = parsed.frequency.frequencyMetadata + chore.frequency = parsed.frequency.frequency + if (isPlusAccount(userProfile)) { + chore.notification = true + chore.notificationMetadata = { templates: notificationTemplates } + } + } + if (!parsed.frequency && parsed.dueDate) { + chore.nextDueDate = new Date(parsed.dueDate).toISOString() + chore.notificationMetadata = { templates: notificationTemplates } + } + + return chore +} diff --git a/src/views/components/VoiceToTask/useVoiceToTask.js b/src/views/components/VoiceToTask/useVoiceToTask.js new file mode 100644 index 0000000..45cf0fe --- /dev/null +++ b/src/views/components/VoiceToTask/useVoiceToTask.js @@ -0,0 +1,217 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { voiceInputService } from '../../../service/VoiceInputService' +import { generateUUID } from '../../../utils/UUID' +import { + applyScratchThat, + normalizeSpokenText, + splitSpokenSegments, +} from './voiceNormalizer' + +// Mic gesture: hold = push-to-talk (release stops), quick tap = hands-free +// lock (tap again to stop). In hands-free mode, sustained silence auto-stops +// into review so the user is never stuck watching a live mic. + +const TAP_THRESHOLD_MS = 400 +const HANDS_FREE_SILENCE_STOP_MS = 8000 +const HANDS_FREE_EMPTY_STOP_MS = 20000 + +const haptic = async kind => { + try { + const { Haptics, ImpactStyle, NotificationType } = await import( + '@capacitor/haptics' + ) + if (kind === 'notification') { + await Haptics.notification({ type: NotificationType.Success }) + } else if (kind === 'medium') { + await Haptics.impact({ style: ImpactStyle.Medium }) + } else { + await Haptics.impact({ style: ImpactStyle.Light }) + } + } catch { + // no haptics on this platform + } +} + +// phases: idle | listening | review | denied +export function useVoiceToTask({ members = [] } = {}) { + const [phase, setPhase] = useState('idle') + const [isLocked, setIsLocked] = useState(false) + const [partialText, setPartialText] = useState('') + const [segments, setSegments] = useState([]) + + // Kept in sync manually (not via render) so segment commits that happen + // inside voiceInputService.stop() are visible immediately afterwards. + const segmentsRef = useRef([]) + const membersRef = useRef(members) + membersRef.current = members + + const phaseRef = useRef(phase) + phaseRef.current = phase + const lockedRef = useRef(isLocked) + lockedRef.current = isLocked + + const pressStartedAtRef = useRef(0) + const pressStartedListeningRef = useRef(false) + const lastActivityRef = useRef(0) + const watchdogRef = useRef(null) + + const applySegments = useCallback(next => { + segmentsRef.current = next + setSegments(next) + }, []) + + const commitSegment = useCallback( + rawText => { + const normalized = normalizeSpokenText(rawText, { + members: membersRef.current, + }) + const { text, dropPrevious } = applyScratchThat(normalized) + const pieces = splitSpokenSegments(text) + if (!dropPrevious && pieces.length === 0) return + + let base = segmentsRef.current + if (dropPrevious && base.length > 0) { + base = base.slice(0, -1) + haptic('medium') + } + if (pieces.length > 0) haptic('light') + applySegments([ + ...base, + ...pieces.map(piece => ({ id: generateUUID(), text: piece })), + ]) + }, + [applySegments], + ) + + const stopListening = useCallback(async () => { + if (watchdogRef.current) { + clearInterval(watchdogRef.current) + watchdogRef.current = null + } + await voiceInputService.stop() + setPartialText('') + setIsLocked(false) + // stop() commits any buffered partial synchronously through onSegment, + // so the ref is up to date by the time we read it + setPhase(segmentsRef.current.length > 0 ? 'review' : 'idle') + haptic('light') + }, []) + + const startListening = useCallback(async () => { + const permission = await voiceInputService.requestPermission() + if (permission !== 'granted') { + setPhase('denied') + return false + } + lastActivityRef.current = Date.now() + await voiceInputService.start({ + onPartial: text => { + lastActivityRef.current = Date.now() + setPartialText( + normalizeSpokenText(text, { members: membersRef.current }), + ) + }, + onSegment: commitSegment, + onError: () => { + setPhase('denied') + }, + onStateChange: () => {}, + }) + setPhase('listening') + haptic('medium') + + // Hands-free: auto-stop into review after sustained silence + watchdogRef.current = setInterval(() => { + if (phaseRef.current !== 'listening' || !lockedRef.current) return + const idleFor = Date.now() - lastActivityRef.current + const limit = + segmentsRef.current.length > 0 + ? HANDS_FREE_SILENCE_STOP_MS + : HANDS_FREE_EMPTY_STOP_MS + if (idleFor > limit) { + stopListening() + } + }, 1000) + return true + }, [commitSegment, stopListening]) + + const micPressDown = useCallback(() => { + pressStartedAtRef.current = Date.now() + if (phaseRef.current === 'listening') { + pressStartedListeningRef.current = false + return + } + pressStartedListeningRef.current = true + startListening() + }, [startListening]) + + const micPressUp = useCallback(() => { + const held = Date.now() - pressStartedAtRef.current + if (pressStartedListeningRef.current) { + if (held < TAP_THRESHOLD_MS) { + // Quick tap → hands-free lock + setIsLocked(true) + } else { + // Hold-to-talk → release ends the capture + stopListening() + } + } else if (phaseRef.current === 'listening') { + // Tap while already listening (locked mode) → stop + stopListening() + } + pressStartedListeningRef.current = false + }, [stopListening]) + + const removeSegment = useCallback( + id => { + applySegments(segmentsRef.current.filter(s => s.id !== id)) + }, + [applySegments], + ) + + const updateSegment = useCallback( + (id, text) => { + applySegments( + segmentsRef.current + .map(s => (s.id === id ? { ...s, text: text.trim() } : s)) + .filter(s => s.text), + ) + }, + [applySegments], + ) + + const reset = useCallback(() => { + voiceInputService.stop() + if (watchdogRef.current) { + clearInterval(watchdogRef.current) + watchdogRef.current = null + } + applySegments([]) + setPartialText('') + setIsLocked(false) + setPhase('idle') + }, [applySegments]) + + // Stop the recognizer if the panel unmounts mid-capture + useEffect(() => { + return () => { + voiceInputService.stop() + if (watchdogRef.current) clearInterval(watchdogRef.current) + } + }, []) + + return { + phase, + isLocked, + partialText, + segments, + micPressDown, + micPressUp, + startListening, + stopListening, + removeSegment, + updateSegment, + reset, + isNative: voiceInputService.isNative, + } +} diff --git a/src/views/components/VoiceToTask/voiceNormalizer.js b/src/views/components/VoiceToTask/voiceNormalizer.js new file mode 100644 index 0000000..1fffc8d --- /dev/null +++ b/src/views/components/VoiceToTask/voiceNormalizer.js @@ -0,0 +1,129 @@ +// Deterministic transforms that turn spoken language into the typed syntax +// CustomParsers understands. No LLM — instant, predictable, fully offline. +// +// "label groceries" → "#groceries" +// "assign to Sarah" → "@Sarah" (only when Sarah is a circle member) +// "worth five points" → "*5" +// "p one" / "top priority" → "priority 1" (parsePriority already handles that) + +const FILLER_REGEX = /(?:^|\s)(?:um+|uh+|erm+|hmm+|mmm+)(?=[\s,.!?]|$)[,.]?/gi + +const NUMBER_WORDS = { + one: 1, + two: 2, + three: 3, + four: 4, + five: 5, + six: 6, + seven: 7, + eight: 8, + nine: 9, + ten: 10, + fifteen: 15, + twenty: 20, + 'twenty five': 25, + 'twenty-five': 25, + fifty: 50, + hundred: 100, + 'one hundred': 100, +} + +const NUMBER_WORD_PATTERN = Object.keys(NUMBER_WORDS) + // Longest first so "twenty five" wins over "five" + .sort((a, b) => b.length - a.length) + .join('|') + +const escapeRegex = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + +export const stripFillers = text => + text.replace(FILLER_REGEX, ' ').replace(/\s+/g, ' ').trim() + +const normalizePriority = text => + text + .replace(/\b(?:top|highest)\s+priority\b/gi, 'priority 1') + .replace( + /\bp[\s-]?(one|two|three|four|[1-4])\b/gi, + (_, n) => `priority ${NUMBER_WORDS[n.toLowerCase()] || n}`, + ) + +const normalizePoints = text => + text.replace( + new RegExp( + `\\b(?:worth\\s+)?(\\d+|${NUMBER_WORD_PATTERN})\\s+points?\\b`, + 'gi', + ), + (_, n) => `*${NUMBER_WORDS[n.toLowerCase()] || n} points`, + ) + +const normalizeLabels = text => + text.replace( + /\b(?:with\s+)?(?:hash\s?tag|labell?ed(?:\s+as)?|label|tagged(?:\s+as)?|tag)\s+([\p{L}\p{N}_]+)/giu, + '#$1', + ) + +const normalizeAssignees = (text, members = []) => { + const assignVerb = '(?:assign(?:ed)?\\s+(?:this\\s+|it\\s+)?to|for)' + let out = text.replace( + new RegExp(`\\b${assignVerb}\\s+(?:anyone|anybody|everyone)\\b`, 'gi'), + '@Anyone', + ) + + for (const member of members) { + const displayName = member.displayName + if (!displayName) continue + const firstName = displayName.split(/\s+/)[0] + // Full display name first so "assign to Mo Tarbin" doesn't leave "Tarbin" + const names = [...new Set([displayName, firstName])].filter( + n => n.length > 1, + ) + for (const name of names) { + out = out.replace( + new RegExp(`\\b${assignVerb}\\s+${escapeRegex(name)}\\b`, 'gi'), + `@${displayName}`, + ) + } + } + return out +} + +export const normalizeSpokenText = (text, { members = [] } = {}) => { + let out = stripFillers(text) + out = normalizePriority(out) + out = normalizePoints(out) + out = normalizeLabels(out) + out = normalizeAssignees(out, members) + return out.replace(/\s+/g, ' ').trim() +} + +// ── Multi-task segmentation ───────────────────────────────────────────────── +// A pause (utterance boundary) always splits — that's handled upstream by the +// recognizer. These spoken separators additionally split within one utterance. +// Deliberately conservative: "and then" is NOT a separator ("wash and then +// fold laundry" is one task). + +const SEPARATOR_REGEX = + /\s*\b(?:and\s+also|also|next\s+task|new\s+task|another\s+task)\b[,.]?\s*/gi + +export const splitSpokenSegments = text => + text + .split(SEPARATOR_REGEX) + .map(s => s.trim().replace(/^[,.]\s*/, '')) + .filter(Boolean) + +// ── "Scratch that" correction ─────────────────────────────────────────────── +// Everything spoken before the command dies. If the command opens the +// utterance ("…pause… scratch that"), the previously committed task dies +// instead. Words after the command carry on as the replacement. + +const SCRATCH_REGEX = + /\s*\b(?:(?:scratch|forget|delete|remove|cancel)\s+(?:that|this|it|last(?:\s+one)?)|never\s?mind)\b[,.]?\s*/gi + +export const applyScratchThat = text => { + const parts = text.split(SCRATCH_REGEX) + if (parts.length === 1) { + return { text: text.trim(), dropPrevious: false } + } + const before = parts.slice(0, -1).join(' ').trim() + const after = parts[parts.length - 1].trim() + return { text: after, dropPrevious: before.length === 0 } +}