Add transliteration support for non-latin scripts

data/applications/src/main/java/de/mm20/launcher2/applications/AppRepository.kt

@@ -10,12 +10,12 @@ import android.os.Handler
 import android.os.Looper
 import android.os.Process
 import android.os.UserHandle
-import de.mm20.launcher2.ktx.normalize
 import de.mm20.launcher2.profiles.Profile
 import de.mm20.launcher2.profiles.ProfileManager
 import de.mm20.launcher2.search.Application
 import de.mm20.launcher2.search.ResultScore
 import de.mm20.launcher2.search.SearchableRepository
+import de.mm20.launcher2.search.StringNormalizer
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.toImmutableList
 import kotlinx.coroutines.CoroutineScope
@@ -30,8 +30,6 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
-import org.apache.commons.text.similarity.FuzzyScore
-import java.util.Locale
 
 interface AppRepository : SearchableRepository<Application> {
     fun findOne(
@@ -45,6 +43,7 @@ interface AppRepository : SearchableRepository<Application> {
 internal class AppRepositoryImpl(
     private val context: Context,
     private val profileManager: ProfileManager,
+    private val stringNormalizer: StringNormalizer,
 ) : AppRepository {
     private val scope = CoroutineScope(Dispatchers.Default + Job())
 
@@ -240,6 +239,7 @@ internal class AppRepositoryImpl(
     }
 
     override fun search(query: String, allowNetwork: Boolean): Flow<ImmutableList<LauncherApp>> {
+        val normalizedQuery = stringNormalizer.normalize(query)
         return installedApps.map { apps ->
             withContext(Dispatchers.Default) {
                 val appResults = mutableListOf<LauncherApp>()
@@ -247,9 +247,11 @@ internal class AppRepositoryImpl(
                     appResults.addAll(apps)
                 } else {
                     appResults.addAll(apps.mapNotNull {
-                        val score = ResultScore(
-                            query = query,
-                            primaryFields = listOf(it.label),
+                        val score = ResultScore.from(
+                            query = normalizedQuery,
+                            primaryFields = listOf(
+                                stringNormalizer.normalize(it.label)
+                            ),
                         )
                         if (score.score < 0.8f) return@mapNotNull null
                         it.copy(

data/applications/src/main/java/de/mm20/launcher2/applications/AppSerialization.kt

@@ -12,6 +12,7 @@ import de.mm20.launcher2.ktx.isAtLeastApiLevel
 import de.mm20.launcher2.search.SavableSearchable
 import de.mm20.launcher2.search.SearchableDeserializer
 import de.mm20.launcher2.search.SearchableSerializer
+import de.mm20.launcher2.search.StringNormalizer
 import org.json.JSONObject
 
 internal class LockedPrivateProfileAppSerializer : SearchableSerializer {

data/applications/src/main/java/de/mm20/launcher2/applications/LauncherApp.kt

@@ -1,6 +1,5 @@
 package de.mm20.launcher2.applications
 
-import android.app.admin.DevicePolicyManager
 import android.content.ActivityNotFoundException
 import android.content.ComponentName
 import android.content.Context
@@ -33,7 +32,6 @@ import de.mm20.launcher2.search.SearchableSerializer
 import de.mm20.launcher2.search.StoreLink
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
-import kotlin.math.roundToInt
 
 internal data class LauncherApp(
     private val launcherActivityInfo: LauncherActivityInfo,
@@ -50,9 +48,16 @@ internal data class LauncherApp(
     override val label: String = launcherActivityInfo.label.toString()
 
 
-    constructor(context: Context, launcherActivityInfo: LauncherActivityInfo, score: ResultScore = ResultScore.Unspecified) : this(
+    constructor(
+        context: Context,
+        launcherActivityInfo: LauncherActivityInfo,
+        score: ResultScore = ResultScore.Unspecified,
+    ) : this(
         launcherActivityInfo,
-        versionName = getPackageVersionName(context, launcherActivityInfo.applicationInfo.packageName),
+        versionName = getPackageVersionName(
+            context,
+            launcherActivityInfo.applicationInfo.packageName
+        ),
         isSuspended = launcherActivityInfo.applicationInfo.flags and ApplicationInfo.FLAG_SUSPENDED != 0,
         userSerialNumber = launcherActivityInfo.user.getSerialNumber(context),
         score = score,
@@ -63,7 +68,8 @@ internal data class LauncherApp(
 
     private val isMainProfile = launcherActivityInfo.user == Process.myUserHandle()
 
-    private val isSystemApp: Boolean = launcherActivityInfo.applicationInfo.flags and ApplicationInfo.FLAG_SYSTEM != 0
+    private val isSystemApp: Boolean =
+        launcherActivityInfo.applicationInfo.flags and ApplicationInfo.FLAG_SYSTEM != 0
 
     override val canUninstall: Boolean
         get() = !isSystemApp && isMainProfile
@@ -89,7 +95,7 @@ internal data class LauncherApp(
         try {
             val icon =
                 withContext(Dispatchers.IO) {
-                    val density = size / (108/1.5)
+                    val density = size / (108 / 1.5)
                     launcherActivityInfo.getIcon(0)
 
                 } ?: return null
@@ -269,7 +275,10 @@ internal data class LauncherApp(
 
         fun isSuspended(context: Context, packageName: String): Boolean {
             return try {
-                context.packageManager.getApplicationInfo(packageName, 0).flags and ApplicationInfo.FLAG_SUSPENDED != 0
+                context.packageManager.getApplicationInfo(
+                    packageName,
+                    0
+                ).flags and ApplicationInfo.FLAG_SUSPENDED != 0
             } catch (e: PackageManager.NameNotFoundException) {
                 false
             }

data/applications/src/main/java/de/mm20/launcher2/applications/Module.kt

@@ -9,6 +9,6 @@ import org.koin.dsl.module
 
 val applicationsModule = module {
     factory<SearchableRepository<Application>>(named<Application>()) { get<AppRepository>() }
-    single<AppRepository> { AppRepositoryImpl(androidContext(), get()) }
+    single<AppRepository> { AppRepositoryImpl(androidContext(), get(), get()) }
     factory<SearchableDeserializer>(named(LauncherApp.Domain)) { LauncherAppDeserializer(androidContext()) }
 }

data/appshortcuts/src/main/java/de/mm20/launcher2/appshortcuts/AppShortcutConfigActivity.kt

@@ -6,7 +6,6 @@ import android.content.pm.LauncherActivityInfo
 import android.content.pm.LauncherApps
 import android.graphics.drawable.Drawable
 import androidx.core.content.getSystemService
-import de.mm20.launcher2.ktx.romanize
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
 import java.text.Collator
@@ -30,6 +29,6 @@ class AppShortcutConfigActivity(
         val label1 = label
         val label2 = other.label
         return Collator.getInstance().apply { strength = Collator.SECONDARY }
-            .compare(label1.romanize(), label2.romanize())
+            .compare(label1, label2)
     }
 }

data/appshortcuts/src/main/java/de/mm20/launcher2/appshortcuts/AppShortcutRepository.kt

@@ -9,7 +9,6 @@ import android.os.Looper
 import android.os.Process
 import android.os.UserHandle
 import androidx.core.content.getSystemService
-import de.mm20.launcher2.ktx.normalize
 import de.mm20.launcher2.permissions.PermissionGroup
 import de.mm20.launcher2.permissions.PermissionsManager
 import de.mm20.launcher2.preferences.search.ShortcutSearchSettings
@@ -17,6 +16,7 @@ import de.mm20.launcher2.profiles.ProfileManager
 import de.mm20.launcher2.search.AppShortcut
 import de.mm20.launcher2.search.ResultScore
 import de.mm20.launcher2.search.SearchableRepository
+import de.mm20.launcher2.search.StringNormalizer
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toImmutableList
@@ -34,8 +34,6 @@ import kotlinx.coroutines.flow.flowOf
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.shareIn
 import kotlinx.coroutines.withContext
-import org.apache.commons.text.similarity.FuzzyScore
-import java.util.Locale
 
 interface AppShortcutRepository : SearchableRepository<AppShortcut> {
 
@@ -57,6 +55,7 @@ internal class AppShortcutRepositoryImpl(
     private val permissionsManager: PermissionsManager,
     private val settings: ShortcutSearchSettings,
     private val profileManager: ProfileManager,
+    private val stringNormalizer: StringNormalizer,
 ) : AppShortcutRepository {
 
     private val scope = CoroutineScope(Dispatchers.Default + Job())
@@ -120,6 +119,8 @@ internal class AppShortcutRepositoryImpl(
             return flowOf(persistentListOf())
         }
 
+        val normalizedQuery = stringNormalizer.normalize(query)
+
         return combine(
             listOf(
                 settings.enabled,
@@ -146,9 +147,14 @@ internal class AppShortcutRepositoryImpl(
                     )
                     val shortcuts = launcherApps.getShortcuts(shortcutQuery, Process.myUserHandle())
                         ?.mapNotNull {
-                            val score = ResultScore(
-                                query = query,
-                                primaryFields = listOfNotNull(it.longLabel?.toString(), it.shortLabel?.toString())
+                            val score = ResultScore.from(
+                                query = normalizedQuery,
+                                primaryFields = listOfNotNull(
+                                    it.longLabel?.toString()
+                                        ?.let { stringNormalizer.normalize(it) },
+                                    it.shortLabel?.toString()
+                                        ?.let { stringNormalizer.normalize(it) },
+                                )
                             )
                             if (score.score < 0.8f) return@mapNotNull null
                             LauncherShortcut(
@@ -227,13 +233,4 @@ internal class AppShortcutRepositoryImpl(
         }
         return results.sorted()
     }
-
-
-    private fun matches(label: String, query: String): Boolean {
-        val normalizedLabel = label.normalize()
-        val normalizedQuery = query.normalize()
-        if (normalizedLabel.contains(normalizedQuery)) return true
-        val fuzzyScore = FuzzyScore(Locale.getDefault())
-        return fuzzyScore.fuzzyScore(normalizedLabel, normalizedQuery) >= query.length * 1.5
-    }
 }

data/appshortcuts/src/main/java/de/mm20/launcher2/appshortcuts/Module.kt

@@ -8,7 +8,7 @@ import org.koin.core.qualifier.named
 import org.koin.dsl.module
 
 val appShortcutsModule = module {
-    factory<AppShortcutRepository> { AppShortcutRepositoryImpl(androidContext(), get(), get(), get()) }
+    factory<AppShortcutRepository> { AppShortcutRepositoryImpl(androidContext(), get(), get(), get(), get()) }
     factory<SearchableRepository<AppShortcut>>(named<AppShortcut>()) { get<AppShortcutRepository>() }
     factory<SearchableDeserializer>(named(LauncherShortcut.Domain)) { LauncherShortcutDeserializer(androidContext()) }
     factory<SearchableDeserializer>(named(LegacyShortcut.Domain)) { LegacyShortcutDeserializer(androidContext()) }

data/i18n/.gitignore

@@ -0,0 +1 @@
+/build

data/i18n/build.gradle.kts

@@ -0,0 +1,52 @@
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+
+plugins {
+    alias(libs.plugins.android.library)
+    alias(libs.plugins.kotlin.android)
+    alias(libs.plugins.kotlin.plugin.serialization)
+}
+
+android {
+    compileSdk = libs.versions.compileSdk.get().toInt()
+
+    defaultConfig {
+        minSdk = libs.versions.minSdk.get().toInt()
+
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+        consumerProguardFiles("consumer-rules.pro")
+    }
+
+    buildTypes {
+        release {
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+
+    kotlin {
+        compilerOptions {
+            jvmTarget.set(JvmTarget.JVM_1_8)
+        }
+    }
+    namespace = "de.mm20.launcher2.data.i18n"
+}
+
+dependencies {
+    implementation(libs.bundles.kotlin)
+
+    implementation(libs.koin.android)
+    implementation(libs.commons.text)
+
+    implementation(project(":core:ktx"))
+    implementation(project(":core:base"))
+    implementation(project(":core:crashreporter"))
+    implementation(project(":core:preferences"))
+
+}

data/i18n/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

data/i18n/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+</manifest>

data/i18n/src/main/java/de/mm20/launcher2/data/CompatStringNormalizer.kt

@@ -0,0 +1,17 @@
+package de.mm20.launcher2.data
+
+import de.mm20.launcher2.search.StringNormalizer
+import org.apache.commons.lang3.StringUtils
+import java.util.Locale
+
+/**
+ * Pre Android 10 StringNormalizer. Only strips accents from latin characters
+ */
+internal class CompatStringNormalizer: StringNormalizer  {
+    override fun normalize(input: String): String {
+        return StringUtils.stripAccents(input.lowercase(Locale.getDefault()))
+            .replace("æ", "ae")
+            .replace("œ", "oe")
+            .replace("ß", "ss")
+    }
+}

data/i18n/src/main/java/de/mm20/launcher2/data/IcuStringNormalizer.kt

@@ -0,0 +1,113 @@
+package de.mm20.launcher2.data
+
+import android.content.Context
+import android.icu.text.Transliterator
+import android.icu.util.ULocale
+import androidx.annotation.RequiresApi
+import de.mm20.launcher2.crashreporter.CrashReporter
+import de.mm20.launcher2.preferences.ui.LocaleSettings
+import de.mm20.launcher2.search.StringNormalizer
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.flow.SharingStarted
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.stateIn
+import org.apache.commons.lang3.StringUtils
+import java.util.Locale
+
+@RequiresApi(29)
+internal class IcuStringNormalizer(
+    private val context: Context,
+    localeSettings: LocaleSettings,
+) : StringNormalizer {
+
+    private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+
+    private val transliterator = localeSettings.transliterator
+        .map {
+            try {
+                getTransliterator(it)
+            } catch (e: IllegalArgumentException) {
+                CrashReporter.logException(e)
+                null
+            }
+        }
+        .stateIn(scope, SharingStarted.Eagerly, null)
+
+    override fun normalize(input: String): String {
+        val transliterator = transliterator.value
+
+        if (transliterator ==  null) {
+            return StringUtils.stripAccents(input.lowercase(Locale.getDefault()))
+                .replace("æ", "ae")
+                .replace("œ", "oe")
+                .replace("ß", "ss")
+        }
+
+        return transliterator.transliterate(input).lowercase()
+    }
+
+    private fun getTransliterator(preferenceValue: String?): Transliterator {
+        val id = preferenceValue ?: return Transliterator.getInstance(DisabledTransliteratorId)
+
+        if (id.isNotBlank()) {
+            return Transliterator.getInstance("$id;$BaseTransliteratorId")
+        }
+
+        val locales = context.resources.configuration.locales
+
+        if (locales.isEmpty) {
+            Transliterator.getInstance(BaseTransliteratorId)
+        }
+
+        val scripts = mutableSetOf<String>()
+        val languages = mutableSetOf<String>()
+
+        val availableIds = Transliterator.getAvailableIDs().toList()
+
+        for (i in 0..<locales.size()) {
+            val locale = locales.get(i)
+            val ulocale = ULocale.addLikelySubtags(ULocale.forLocale(locale))
+
+            val lng = ulocale.language
+            val scr = ulocale.script
+
+            if (!languages.contains(lng)) {
+                val filter = "${lng}-${lng}_Latn"
+
+                val id = availableIds.find { it.startsWith(filter) }
+
+                if (id != null) {
+                    return Transliterator.getInstance("$id;$BaseTransliteratorId")
+                }
+
+                languages.add(lng)
+            }
+
+            if (!scripts.contains(ulocale.script)) {
+                val filter = "${scr}-Latn"
+
+                val id = availableIds.find { it.startsWith(filter) }
+
+                if (id != null) {
+                    return Transliterator.getInstance("$id;$BaseTransliteratorId")
+                }
+                scripts.add(ulocale.script)
+            }
+        }
+        return Transliterator.getInstance(BaseTransliteratorId)
+    }
+
+    companion object {
+        /**
+         * Transliterator that is used when transliteration is disabled
+         */
+        private const val DisabledTransliteratorId = "Latin-ASCII"
+
+        /**
+         * Transliterator that is used when no script or language is specified
+         */
+        private const val BaseTransliteratorId = "Any-Latin;Latin-ASCII"
+    }
+}

data/i18n/src/main/java/de/mm20/launcher2/data/Module.kt

@@ -0,0 +1,13 @@
+package de.mm20.launcher2.data
+
+import de.mm20.launcher2.ktx.isAtLeastApiLevel
+import de.mm20.launcher2.search.StringNormalizer
+import org.koin.android.ext.koin.androidContext
+import org.koin.dsl.module
+
+val i18nDataModule = module {
+    single<StringNormalizer> {
+        if (isAtLeastApiLevel(29)) IcuStringNormalizer(androidContext(), get())
+        else CompatStringNormalizer()
+    }
+}

data/locations/src/main/java/de/mm20/launcher2/locations/providers/openstreetmaps/OsmLocationProvider.kt

@@ -214,7 +214,7 @@ internal class OsmLocationProvider(
 
     private fun delocalizeToQueryableTags(localizedQuery: String): List<String> =
         poiCategories.flatMap { (string, tags) ->
-            val score = ResultScore(
+            val score = ResultScore.from(
                 localizedQuery,
                 primaryFields = listOf(string)
             )