Add transliteration support for non-latin scripts

data/i18n/.gitignore

@@ -0,0 +1 @@
+/build

data/i18n/build.gradle.kts

@@ -0,0 +1,52 @@
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+
+plugins {
+    alias(libs.plugins.android.library)
+    alias(libs.plugins.kotlin.android)
+    alias(libs.plugins.kotlin.plugin.serialization)
+}
+
+android {
+    compileSdk = libs.versions.compileSdk.get().toInt()
+
+    defaultConfig {
+        minSdk = libs.versions.minSdk.get().toInt()
+
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+        consumerProguardFiles("consumer-rules.pro")
+    }
+
+    buildTypes {
+        release {
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+
+    kotlin {
+        compilerOptions {
+            jvmTarget.set(JvmTarget.JVM_1_8)
+        }
+    }
+    namespace = "de.mm20.launcher2.data.i18n"
+}
+
+dependencies {
+    implementation(libs.bundles.kotlin)
+
+    implementation(libs.koin.android)
+    implementation(libs.commons.text)
+
+    implementation(project(":core:ktx"))
+    implementation(project(":core:base"))
+    implementation(project(":core:crashreporter"))
+    implementation(project(":core:preferences"))
+
+}

data/i18n/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

data/i18n/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+</manifest>

data/i18n/src/main/java/de/mm20/launcher2/data/CompatStringNormalizer.kt

@@ -0,0 +1,17 @@
+package de.mm20.launcher2.data
+
+import de.mm20.launcher2.search.StringNormalizer
+import org.apache.commons.lang3.StringUtils
+import java.util.Locale
+
+/**
+ * Pre Android 10 StringNormalizer. Only strips accents from latin characters
+ */
+internal class CompatStringNormalizer: StringNormalizer  {
+    override fun normalize(input: String): String {
+        return StringUtils.stripAccents(input.lowercase(Locale.getDefault()))
+            .replace("æ", "ae")
+            .replace("œ", "oe")
+            .replace("ß", "ss")
+    }
+}

data/i18n/src/main/java/de/mm20/launcher2/data/IcuStringNormalizer.kt

@@ -0,0 +1,113 @@
+package de.mm20.launcher2.data
+
+import android.content.Context
+import android.icu.text.Transliterator
+import android.icu.util.ULocale
+import androidx.annotation.RequiresApi
+import de.mm20.launcher2.crashreporter.CrashReporter
+import de.mm20.launcher2.preferences.ui.LocaleSettings
+import de.mm20.launcher2.search.StringNormalizer
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.flow.SharingStarted
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.stateIn
+import org.apache.commons.lang3.StringUtils
+import java.util.Locale
+
+@RequiresApi(29)
+internal class IcuStringNormalizer(
+    private val context: Context,
+    localeSettings: LocaleSettings,
+) : StringNormalizer {
+
+    private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+
+    private val transliterator = localeSettings.transliterator
+        .map {
+            try {
+                getTransliterator(it)
+            } catch (e: IllegalArgumentException) {
+                CrashReporter.logException(e)
+                null
+            }
+        }
+        .stateIn(scope, SharingStarted.Eagerly, null)
+
+    override fun normalize(input: String): String {
+        val transliterator = transliterator.value
+
+        if (transliterator ==  null) {
+            return StringUtils.stripAccents(input.lowercase(Locale.getDefault()))
+                .replace("æ", "ae")
+                .replace("œ", "oe")
+                .replace("ß", "ss")
+        }
+
+        return transliterator.transliterate(input).lowercase()
+    }
+
+    private fun getTransliterator(preferenceValue: String?): Transliterator {
+        val id = preferenceValue ?: return Transliterator.getInstance(DisabledTransliteratorId)
+
+        if (id.isNotBlank()) {
+            return Transliterator.getInstance("$id;$BaseTransliteratorId")
+        }
+
+        val locales = context.resources.configuration.locales
+
+        if (locales.isEmpty) {
+            Transliterator.getInstance(BaseTransliteratorId)
+        }
+
+        val scripts = mutableSetOf<String>()
+        val languages = mutableSetOf<String>()
+
+        val availableIds = Transliterator.getAvailableIDs().toList()
+
+        for (i in 0..<locales.size()) {
+            val locale = locales.get(i)
+            val ulocale = ULocale.addLikelySubtags(ULocale.forLocale(locale))
+
+            val lng = ulocale.language
+            val scr = ulocale.script
+
+            if (!languages.contains(lng)) {
+                val filter = "${lng}-${lng}_Latn"
+
+                val id = availableIds.find { it.startsWith(filter) }
+
+                if (id != null) {
+                    return Transliterator.getInstance("$id;$BaseTransliteratorId")
+                }
+
+                languages.add(lng)
+            }
+
+            if (!scripts.contains(ulocale.script)) {
+                val filter = "${scr}-Latn"
+
+                val id = availableIds.find { it.startsWith(filter) }
+
+                if (id != null) {
+                    return Transliterator.getInstance("$id;$BaseTransliteratorId")
+                }
+                scripts.add(ulocale.script)
+            }
+        }
+        return Transliterator.getInstance(BaseTransliteratorId)
+    }
+
+    companion object {
+        /**
+         * Transliterator that is used when transliteration is disabled
+         */
+        private const val DisabledTransliteratorId = "Latin-ASCII"
+
+        /**
+         * Transliterator that is used when no script or language is specified
+         */
+        private const val BaseTransliteratorId = "Any-Latin;Latin-ASCII"
+    }
+}

data/i18n/src/main/java/de/mm20/launcher2/data/Module.kt

@@ -0,0 +1,13 @@
+package de.mm20.launcher2.data
+
+import de.mm20.launcher2.ktx.isAtLeastApiLevel
+import de.mm20.launcher2.search.StringNormalizer
+import org.koin.android.ext.koin.androidContext
+import org.koin.dsl.module
+
+val i18nDataModule = module {
+    single<StringNormalizer> {
+        if (isAtLeastApiLevel(29)) IcuStringNormalizer(androidContext(), get())
+        else CompatStringNormalizer()
+    }
+}