From 108376e0a044a4ca587c1f66acbe5f303fd8bc90 Mon Sep 17 00:00:00 2001 From: davecraig <8530624+davecraig@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:54:21 +0000 Subject: [PATCH 01/18] Bump version to 0.3.11, version code 173 --- app/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 9bf2e70b..abb057d3 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -47,8 +47,8 @@ android { applicationId = "org.scottishtecharmy.soundscape" minSdk = 30 targetSdk = 35 - versionCode = 172 - versionName = "0.3.10" + versionCode = 173 + versionName = "0.3.11" // Maintaining this list means that we can exclude translations that aren't complete yet resourceConfigurations.addAll(listOf( From 6f900dc69e9043de82b340329dce15d0615ffb25 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Mon, 2 Mar 2026 10:15:25 +0000 Subject: [PATCH 02/18] Add commented out before and after testing for grid processing --- app/.gitignore | 4 ++- .../soundscape/MvtTileTest.kt | 27 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/app/.gitignore b/app/.gitignore index 42afabfd..56dadcc2 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -1 +1,3 @@ -/build \ No newline at end of file +/build +/cache-test +/cache-test2 diff --git a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt index 4201b841..ea81879f 100644 --- a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt +++ b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt @@ -269,7 +269,7 @@ class MvtTileTest { TreeId.SETTLEMENT_VILLAGE, TreeId.SETTLEMENT_HAMLET, TreeId.TRANSIT -> - {} + {} else -> { for (feature in collection) { @@ -591,7 +591,7 @@ class MvtTileTest { val w1 = 300.0 val w2 = 100.0 val fitness = (w1 * (10 / (10 + sensedRoadInfo.distance))) + - (w2 * (30 / (30 + headingOffSensedRoad))) + (w2 * (30 / (30 + headingOffSensedRoad))) if(fitness > bestFitness) { bestFitness = fitness bestIndex = index @@ -968,18 +968,23 @@ class MvtTileTest { LngLatAlt(location.longitude, location.latitude), emptySet() ) + if(false) { + // This code is useful for comparing before and after changes of grid parsing + val adapter = GeoJsonObjectMoshiAdapter() + val tileOutput = FileOutputStream("cache-test2/output-$x-$y.geojson") + + // Output the GeoJson and check that there's no data left from other tiles. + val collection = FeatureCollection() + for (id in TreeId.entries) { + if (id < TreeId.MAX_COLLECTION_ID) + collection += gridState.getFeatureCollection(id) + } + tileOutput.write(adapter.toJson(collection).toByteArray()) + tileOutput.close() + } } } } -// val adapter = GeoJsonObjectMoshiAdapter() -// val mapMatchingOutput = FileOutputStream("total-output.geojson") -// -// // Output the GeoJson and check that there's no data left from other tiles. -// val collection = gridState.getFeatureCollection(TreeId.WAYS_SELECTION) -// collection += gridState.getFeatureCollection(TreeId.INTERSECTIONS) -// collection += gridState.getFeatureCollection(TreeId.POIS) -// mapMatchingOutput.write(adapter.toJson(collection).toByteArray()) -// mapMatchingOutput.close() } @Test From 44608ccdaf0929e4df61b27b1d4357aeb132b188 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 26 Feb 2026 15:14:38 +0000 Subject: [PATCH 03/18] Initial attempt at speech recognition Pressing the Play/Pause button starts the speech recognition listening. --- app/src/main/AndroidManifest.xml | 4 +- .../soundscape/MainActivity.kt | 9 ++ .../soundscape/SoundscapeServiceConnection.kt | 5 + .../soundscape/screens/home/home/Home.kt | 3 +- .../screens/home/home/HomeContent.kt | 27 +++- .../SoundscapeMediaSessionCallback.kt | 15 ++- .../soundscape/services/SoundscapeService.kt | 73 ++++++++++- .../services/VoiceCommandManager.kt | 115 ++++++++++++++++++ .../soundscape/viewmodels/home/HomeState.kt | 1 + .../viewmodels/home/HomeViewModel.kt | 7 ++ app/src/main/res/values/strings.xml | 13 ++ 11 files changed, 258 insertions(+), 14 deletions(-) create mode 100644 app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 4d81c65a..e6a5832b 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -13,6 +13,8 @@ + + diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt index a4750dfe..be53d10d 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt @@ -96,6 +96,10 @@ class MainActivity : AppCompatActivity() { checkAndRequestLocationPermissions() } + // Microphone permission for voice commands — best-effort; if denied, voice commands are silently skipped + private val micPermissionLauncher = + registerForActivityResult(ActivityResultContracts.RequestPermission()) { /* no-op */ } + // we need location permission to be able to start the service private val locationPermissionRequest = registerForActivityResult( ActivityResultContracts.RequestMultiplePermissions() @@ -746,6 +750,11 @@ class MainActivity : AppCompatActivity() { Log.e(TAG, "startSoundscapeService") val serviceIntent = Intent(this, SoundscapeService::class.java) startForegroundService(serviceIntent) + // Request microphone permission for voice commands (best-effort) + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != android.content.pm.PackageManager.PERMISSION_GRANTED) { + micPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } } companion object { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt index 0326813f..c2bd1ba0 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt @@ -18,6 +18,7 @@ import org.scottishtecharmy.soundscape.services.BeaconState import org.scottishtecharmy.soundscape.services.RoutePlayerState import org.scottishtecharmy.soundscape.services.SoundscapeBinder import org.scottishtecharmy.soundscape.services.SoundscapeService +import org.scottishtecharmy.soundscape.services.VoiceCommandState import javax.inject.Inject @ActivityRetainedScoped @@ -48,6 +49,10 @@ class SoundscapeServiceConnection @Inject constructor() { return soundscapeService?.gridStateFlow } + fun getVoiceCommandStateFlow(): StateFlow? { + return soundscapeService?.voiceCommandStateFlow + } + fun setStreetPreviewMode(on : Boolean, location: LngLatAlt? = null) { Log.d(TAG, "setStreetPreviewMode $on") soundscapeService?.setStreetPreviewMode(on, location) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt index 483910fd..f7c3e0e0 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt @@ -230,7 +230,8 @@ fun Home( goToAppSettings = goToAppSettings, fullscreenMap = fullscreenMap, permissionsRequired = permissionsRequired, - showMap = showMap + showMap = showMap, + voiceCommandListening = state.voiceCommandListening ) } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt index 0b5ad0aa..86be79cf 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt @@ -1,6 +1,7 @@ package org.scottishtecharmy.soundscape.screens.home.home import android.content.Context +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -8,6 +9,7 @@ import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.defaultMinSize +import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding @@ -160,14 +162,16 @@ fun HomeContent( goToAppSettings: (Context) -> Unit, fullscreenMap: MutableState, permissionsRequired: Boolean, - showMap: Boolean) { + showMap: Boolean, + voiceCommandListening: Boolean = false) { val context = LocalContext.current val coroutineScope = rememberCoroutineScope() var fetchingLocation by remember { mutableStateOf(false) } + Box(modifier = modifier.fillMaxSize()) { Column( verticalArrangement = Arrangement.spacedBy(spacing.small), - modifier = modifier + modifier = Modifier.fillMaxSize() ) { if (streetPreviewState.enabled != StreetPreviewEnabled.OFF) { StreetPreview(streetPreviewState, streetPreviewFunctions) @@ -390,6 +394,25 @@ fun HomeContent( } } } + if (voiceCommandListening) { + Box( + contentAlignment = Alignment.BottomCenter, + modifier = Modifier.fillMaxSize() + ) { + Text( + text = stringResource(R.string.voice_cmd_listening), + style = MaterialTheme.typography.bodyLarge, + color = MaterialTheme.colorScheme.onSurface, + modifier = Modifier + .fillMaxWidth() + .background(MaterialTheme.colorScheme.surface.copy(alpha = 0.85f)) + .padding(vertical = spacing.small) + .semantics { liveRegion = LiveRegionMode.Assertive }, + textAlign = TextAlign.Center + ) + } + } + } // end Box } @Preview diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt index 4ee332fa..b3f97674 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt @@ -40,6 +40,9 @@ class SoundscapeMediaSessionCallback(val service : SoundscapeService): KeyEvent::class.java ) + // Ignore key-up events to prevent double-firing + if (keyEvent?.action != KeyEvent.ACTION_DOWN) return false + // So far I've only seen KEYCODE_MEDIA_PLAY_PAUSE, KEYCODE_MEDIA_PREVIOUS and // KEYCODE_MEDIA_NEXT, though that may be specific to my phone. The only event actually // handled for now is KEYCODE_MEDIA_NEXT. @@ -49,11 +52,12 @@ class SoundscapeMediaSessionCallback(val service : SoundscapeService): keyEvent?.let { event -> val decodedKey = when(event.keyCode) { - KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { - // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. - service.routeMute() - "Play/Pause" - } + KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { service.triggerVoiceCommand(); "Play/Pause" } +// KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { +// // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. +// service.routeMute() +// "Play/Pause" +// } KeyEvent.KEYCODE_MEDIA_SKIP_FORWARD -> { // ⏩ Skip Forward if(!service.routeSkipNext()) { @@ -87,7 +91,6 @@ class SoundscapeMediaSessionCallback(val service : SoundscapeService): "Skip backward" } - KeyEvent.KEYCODE_MEDIA_PLAY -> "Play" KeyEvent.KEYCODE_MEDIA_FAST_FORWARD -> "Fast forward" KeyEvent.KEYCODE_MEDIA_REWIND -> "Rewind" else -> return false diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index a2926697..55896581 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -1,9 +1,11 @@ package org.scottishtecharmy.soundscape.services +import android.Manifest import android.content.Context import android.content.res.Configuration import android.annotation.SuppressLint import android.app.ForegroundServiceStartNotAllowedException +import android.content.pm.PackageManager import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager @@ -23,6 +25,7 @@ import android.widget.Toast import androidx.annotation.OptIn import androidx.core.app.NotificationCompat import androidx.core.app.ServiceCompat +import androidx.core.content.ContextCompat import androidx.media3.common.util.UnstableApi import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService @@ -148,6 +151,11 @@ class SoundscapeService : MediaSessionService() { private val _gridStateFlow = MutableStateFlow(null) var gridStateFlow: StateFlow = _gridStateFlow + // Voice command manager + private lateinit var voiceCommandManager: VoiceCommandManager + val voiceCommandStateFlow: StateFlow + get() = voiceCommandManager.state + // Media control button code private var mediaSession: MediaSession? = null private val mediaPlayer = SoundscapeDummyMediaPlayer() @@ -196,10 +204,11 @@ class SoundscapeService : MediaSessionService() { locationProvider.start(this) directionProvider.start(audioEngine, locationProvider) - val configLocale = getCurrentLocale() - val configuration = Configuration(applicationContext.resources.configuration) - configuration.setLocale(configLocale) - localizedContext = applicationContext.createConfigurationContext(configuration) +// val configLocale = getCurrentLocale() +// val configuration = Configuration(applicationContext.resources.configuration) +// configuration.setLocale(configLocale) +// localizedContext = applicationContext.createConfigurationContext(configuration) +// if (::voiceCommandManager.isInitialized) voiceCommandManager.updateContext(localizedContext) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) } @@ -234,6 +243,7 @@ class SoundscapeService : MediaSessionService() { val configuration = Configuration(applicationContext.resources.configuration) configuration.setLocale(configLocale) localizedContext = applicationContext.createConfigurationContext(configuration) + if (::voiceCommandManager.isInitialized) voiceCommandManager.updateContext(localizedContext) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) started = true } @@ -273,6 +283,12 @@ class SoundscapeService : MediaSessionService() { // create new RealmDB or open existing startRealms(applicationContext) + voiceCommandManager = VoiceCommandManager( + context = this, + onCommand = ::executeVoiceCommand, + onError = { audioEngine.createEarcon(NativeAudioEngine.EARCON_LOW_CONFIDENCE, AudioType.STANDARD) } + ) + mediaSession = MediaSession.Builder(this, mediaPlayer) .setCallback(SoundscapeMediaSessionCallback(this)) .build() @@ -313,6 +329,8 @@ class SoundscapeService : MediaSessionService() { wakeLock?.let { if (it.isHeld) it.release() } wakeLock = null + if (::voiceCommandManager.isInitialized) voiceCommandManager.destroy() + // Clear service reference in binder so that it can be garbage collected binder?.reset() } @@ -505,6 +523,53 @@ class SoundscapeService : MediaSessionService() { } } + fun triggerVoiceCommand() { + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != PackageManager.PERMISSION_GRANTED) return + + if (!requestAudioFocus()) { + Log.w(TAG, "speakText: Could not get audio focus. Aborting callouts.") + return + } + + val ctx = if (::localizedContext.isInitialized) localizedContext else this + + // Clear the text queue + audioEngine.clearTextToSpeechQueue() + // Create the earcon and Listening... speech + audioEngine.createEarcon(NativeAudioEngine.EARCON_CALLOUTS_ON, AudioType.STANDARD) + audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_listening), AudioType.STANDARD) + + // Wait for the TTS to finish before opening the mic + coroutineScope.launch { + val deadline = System.currentTimeMillis() + 3_000L + while (isAudioEngineBusy() && System.currentTimeMillis() < deadline) { + delay(50) + } + withContext(Dispatchers.Main) { + voiceCommandManager.startListening() + } + } + } + + fun executeVoiceCommand(command: VoiceCommand) { + val label = when (command) { + VoiceCommand.MY_LOCATION -> { myLocation(); "My location" } + VoiceCommand.AROUND_ME -> { whatsAroundMe(); "Around me" } + VoiceCommand.AHEAD_OF_ME -> { aheadOfMe(); "Ahead of me" } + VoiceCommand.NEARBY_MARKERS -> { nearbyMarkers(); "Nearby markers" } + VoiceCommand.SKIP_NEXT -> { routeSkipNext(); "Skip next" } + VoiceCommand.SKIP_PREVIOUS -> { routeSkipPrevious(); "Skip previous" } + VoiceCommand.MUTE -> { routeMute(); "Mute" } + VoiceCommand.STOP_ROUTE -> { routeStop(); "Stop route" } + VoiceCommand.UNKNOWN -> { + audioEngine.createEarcon(NativeAudioEngine.EARCON_LOW_CONFIDENCE, AudioType.STANDARD) + null + } + } + label?.let { audioEngine.createTextToSpeech(it, AudioType.STANDARD, 0.0, 0.0, 0.0) } + } + suspend fun searchResult(searchString: String): List? { return geoEngine.searchResult(searchString) } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt new file mode 100644 index 00000000..6d6e620a --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt @@ -0,0 +1,115 @@ +package org.scottishtecharmy.soundscape.services + +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.scottishtecharmy.soundscape.R +import org.scottishtecharmy.soundscape.utils.getCurrentLocale + +sealed class VoiceCommandState { + object Idle : VoiceCommandState() + object Listening : VoiceCommandState() + object Error : VoiceCommandState() +} + +enum class VoiceCommand { + MY_LOCATION, AROUND_ME, AHEAD_OF_ME, NEARBY_MARKERS, + SKIP_NEXT, SKIP_PREVIOUS, MUTE, STOP_ROUTE, UNKNOWN +} + +class VoiceCommandManager( + // Mutable so SoundscapeService can push localizedContext once it's created. + // The service context is used for SpeechRecognizer binding; + // the localized context is used for string-resource keyword lookups. + private var context: Context, + private val onCommand: (VoiceCommand) -> Unit, + private val onError: () -> Unit +) { + + private var speechRecognizer: SpeechRecognizer? = null + private val _state = MutableStateFlow(VoiceCommandState.Idle) + val state: StateFlow = _state.asStateFlow() + + /** Call this whenever SoundscapeService updates its localizedContext. */ + fun updateContext(newContext: Context) { + context = newContext + } + + // Must be called on the main thread (satisfied: service is on main thread) + fun startListening() { + if (_state.value is VoiceCommandState.Listening) return + if (!SpeechRecognizer.isRecognitionAvailable(context)) { onError(); return } + destroyRecognizer() + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) + speechRecognizer?.setRecognitionListener(listener) + val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + // Match recognizer language to the app's configured locale + putExtra(RecognizerIntent.EXTRA_LANGUAGE, getCurrentLocale().toLanguageTag()) + } + speechRecognizer?.startListening(intent) + } + + fun destroy() { + destroyRecognizer() + } + + private fun destroyRecognizer() { + speechRecognizer?.destroy() + speechRecognizer = null + } + + private val listener = object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + _state.value = VoiceCommandState.Listening + } + + override fun onResults(results: Bundle?) { + _state.value = VoiceCommandState.Idle + val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + matches?.forEach { println("speech: $it") } + onCommand(parseCommand(matches?.firstOrNull() ?: "")) + } + + override fun onError(error: Int) { + _state.value = VoiceCommandState.Error + onError() + } + + override fun onEndOfSpeech() {} + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onPartialResults(partialResults: Bundle?) {} + override fun onEvent(eventType: Int, params: Bundle?) {} + } + + private fun parseCommand(text: String): VoiceCommand { + val t = text.lowercase() + + // Match against localized keyword lists stored in string resources. + // Each resource is a comma-separated list of phrases a user might say. + fun matches(resId: Int) = + context.getString(resId).split(",").any { t.contains(it.trim()) } + + return when { + matches(R.string.voice_cmd_my_location) -> VoiceCommand.MY_LOCATION + matches(R.string.voice_cmd_around_me) -> VoiceCommand.AROUND_ME + matches(R.string.voice_cmd_ahead_of_me) -> VoiceCommand.AHEAD_OF_ME + matches(R.string.voice_cmd_nearby_markers) -> VoiceCommand.NEARBY_MARKERS + matches(R.string.voice_cmd_skip_previous) -> VoiceCommand.SKIP_PREVIOUS + matches(R.string.voice_cmd_skip_next) -> VoiceCommand.SKIP_NEXT + matches(R.string.voice_cmd_mute) -> VoiceCommand.MUTE + matches(R.string.voice_cmd_stop_route) -> VoiceCommand.STOP_ROUTE + else -> VoiceCommand.UNKNOWN + } + } +} diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt index 4d21aa58..48f6d633 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt @@ -18,4 +18,5 @@ data class HomeState( var routesTabSelected: Boolean = true, var currentRouteData: RoutePlayerState = RoutePlayerState(), var permissionsRequired: Boolean = false, + var voiceCommandListening: Boolean = false, ) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt index bd3006c9..06155612 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt @@ -20,6 +20,7 @@ import org.scottishtecharmy.soundscape.SoundscapeServiceConnection import org.scottishtecharmy.soundscape.audio.AudioTour import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.VoiceCommandState import javax.inject.Inject @HiltViewModel @@ -106,6 +107,12 @@ class HomeViewModel _state.update { it.copy(currentRouteData = value) } } } + viewModelScope.launch(job!!) { + // Observe voice command listening state + soundscapeServiceConnection.getVoiceCommandStateFlow()?.collectLatest { voiceState -> + _state.update { it.copy(voiceCommandListening = voiceState is VoiceCommandState.Listening) } + } + } } private fun stopMonitoringLocation() { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 6b695714..4f4d8ff2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2576,4 +2576,17 @@ Please report any problems that you have, however small they may be via the *Con Continue Close the popup and continue with the tutorial + + +my location,where am i +around me,what\'s around,around +ahead of me,what\'s ahead,ahead +nearby markers,markers,nearby +skip next,next waypoint,next,skip +skip previous,previous waypoint,previous,go back +mute,silence +stop route,stop + +Listening… From 2304bb33fdaf2633c2a5b01a3fc45d0969a0434e Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 26 Feb 2026 15:14:38 +0000 Subject: [PATCH 04/18] Initial attempt at speech recognition Pressing the Play/Pause button starts the speech recognition listening. A limited set of commands which are all in strings.xml. We get "the whole string" passed in from the speech recognition API so we can match route and marker names too. This could control pretty much everything. We should come up with a design - and it probably goes hand in hand with a menu navigation UI for users whose language is not supported by Android speech recognition. Although these changes should in theory work in other languages, I don't I've figured out how to configure my phone to work and so I get a lot of "Error" callbacks right now. --- app/src/main/AndroidManifest.xml | 3 ++ .../soundscape/services/SoundscapeService.kt | 36 ++++++++++++------- .../services/VoiceCommandManager.kt | 3 +- app/src/main/res/values/strings.xml | 7 ++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index e6a5832b..38924d72 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -148,6 +148,9 @@ + + + \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 55896581..d9d38936 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -286,7 +286,15 @@ class SoundscapeService : MediaSessionService() { voiceCommandManager = VoiceCommandManager( context = this, onCommand = ::executeVoiceCommand, - onError = { audioEngine.createEarcon(NativeAudioEngine.EARCON_LOW_CONFIDENCE, AudioType.STANDARD) } + onError = { + if (requestAudioFocus()) { + audioEngine.createEarcon( + NativeAudioEngine.EARCON_CALLOUTS_OFF, + AudioType.STANDARD + ) + audioEngine.createTextToSpeech("I'm sorry I didn't understand", AudioType.STANDARD) + } + } ) mediaSession = MediaSession.Builder(this, mediaPlayer) @@ -553,21 +561,25 @@ class SoundscapeService : MediaSessionService() { } fun executeVoiceCommand(command: VoiceCommand) { + val ctx = if (::localizedContext.isInitialized) localizedContext else this val label = when (command) { - VoiceCommand.MY_LOCATION -> { myLocation(); "My location" } - VoiceCommand.AROUND_ME -> { whatsAroundMe(); "Around me" } - VoiceCommand.AHEAD_OF_ME -> { aheadOfMe(); "Ahead of me" } - VoiceCommand.NEARBY_MARKERS -> { nearbyMarkers(); "Nearby markers" } - VoiceCommand.SKIP_NEXT -> { routeSkipNext(); "Skip next" } - VoiceCommand.SKIP_PREVIOUS -> { routeSkipPrevious(); "Skip previous" } - VoiceCommand.MUTE -> { routeMute(); "Mute" } - VoiceCommand.STOP_ROUTE -> { routeStop(); "Stop route" } + VoiceCommand.MY_LOCATION -> { myLocation(); null } + VoiceCommand.AROUND_ME -> { whatsAroundMe(); null } + VoiceCommand.AHEAD_OF_ME -> { aheadOfMe(); null } + VoiceCommand.NEARBY_MARKERS -> { nearbyMarkers(); null } + VoiceCommand.SKIP_NEXT -> { routeSkipNext(); null } + VoiceCommand.SKIP_PREVIOUS -> { routeSkipPrevious(); null } + VoiceCommand.MUTE -> { routeMute(); null } + VoiceCommand.STOP_ROUTE -> { routeStop(); null } + VoiceCommand.HELP -> ctx.getString(R.string.voice_cmd_help_response) VoiceCommand.UNKNOWN -> { - audioEngine.createEarcon(NativeAudioEngine.EARCON_LOW_CONFIDENCE, AudioType.STANDARD) - null + audioEngine.createEarcon(NativeAudioEngine.EARCON_CALLOUTS_OFF, AudioType.STANDARD) + "I'm sorry I didn't understand" } } - label?.let { audioEngine.createTextToSpeech(it, AudioType.STANDARD, 0.0, 0.0, 0.0) } + if (requestAudioFocus()) { + label?.let { audioEngine.createTextToSpeech(it, AudioType.STANDARD) } + } } suspend fun searchResult(searchString: String): List? { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt index 6d6e620a..e52813bf 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt @@ -20,7 +20,7 @@ sealed class VoiceCommandState { enum class VoiceCommand { MY_LOCATION, AROUND_ME, AHEAD_OF_ME, NEARBY_MARKERS, - SKIP_NEXT, SKIP_PREVIOUS, MUTE, STOP_ROUTE, UNKNOWN + SKIP_NEXT, SKIP_PREVIOUS, MUTE, STOP_ROUTE, HELP, UNKNOWN } class VoiceCommandManager( @@ -109,6 +109,7 @@ class VoiceCommandManager( matches(R.string.voice_cmd_skip_next) -> VoiceCommand.SKIP_NEXT matches(R.string.voice_cmd_mute) -> VoiceCommand.MUTE matches(R.string.voice_cmd_stop_route) -> VoiceCommand.STOP_ROUTE + matches(R.string.voice_cmd_help) -> VoiceCommand.HELP else -> VoiceCommand.UNKNOWN } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 4f4d8ff2..9d50306c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2580,13 +2580,16 @@ Please report any problems that you have, however small they may be via the *Con my location,where am i -around me,what\'s around,around -ahead of me,what\'s ahead,ahead +"around me,what's around,around" +"ahead of me,what's ahead,ahead" nearby markers,markers,nearby skip next,next waypoint,next,skip skip previous,previous waypoint,previous,go back mute,silence stop route,stop +help,what can I say,commands,what commands + +Available commands: Listening… From c4cc00ad9811c46f2735ea9a24d5fe1177d0897e Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 27 Feb 2026 13:50:08 +0000 Subject: [PATCH 05/18] Add media controls abstraction We've got a number of different ways that media controls can be used, so tidy that away into their own classes. --- .../soundscape/SoundscapeServiceConnection.kt | 2 +- .../services/SoundscapeDummyMediaPlayer.kt | 36 ---------- .../soundscape/services/SoundscapeService.kt | 31 ++++----- .../mediacontrol/MediaControlTarget.kt | 62 +++++++++++++++++ .../SoundscapeDummyMediaPlayer.kt | 69 +++++++++++++++++++ .../SoundscapeMediaSessionCallback.kt | 37 ++++------ .../{ => mediacontrol}/VoiceCommandManager.kt | 20 ++++-- .../viewmodels/home/HomeViewModel.kt | 2 +- 8 files changed, 176 insertions(+), 83 deletions(-) delete mode 100644 app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt create mode 100644 app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt create mode 100644 app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt rename app/src/main/java/org/scottishtecharmy/soundscape/services/{ => mediacontrol}/SoundscapeMediaSessionCallback.kt (69%) rename app/src/main/java/org/scottishtecharmy/soundscape/services/{ => mediacontrol}/VoiceCommandManager.kt (86%) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt index c2bd1ba0..ae36f3cd 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt @@ -18,7 +18,7 @@ import org.scottishtecharmy.soundscape.services.BeaconState import org.scottishtecharmy.soundscape.services.RoutePlayerState import org.scottishtecharmy.soundscape.services.SoundscapeBinder import org.scottishtecharmy.soundscape.services.SoundscapeService -import org.scottishtecharmy.soundscape.services.VoiceCommandState +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import javax.inject.Inject @ActivityRetainedScoped diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt deleted file mode 100644 index 61067709..00000000 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt +++ /dev/null @@ -1,36 +0,0 @@ -package org.scottishtecharmy.soundscape.services - -import android.os.Looper -import androidx.annotation.OptIn -import androidx.media3.common.DeviceInfo -import androidx.media3.common.Player -import androidx.media3.common.SimpleBasePlayer -import androidx.media3.common.util.UnstableApi - -@OptIn(UnstableApi::class) -class SoundscapeDummyMediaPlayer : SimpleBasePlayer(Looper.getMainLooper()) { - - override fun getState(): State { - - val commands : Player.Commands = Player.Commands.EMPTY - commands.buildUpon().addAll( - COMMAND_PLAY_PAUSE, - COMMAND_STOP, - COMMAND_SEEK_TO_NEXT, - COMMAND_SEEK_TO_PREVIOUS, - COMMAND_SEEK_FORWARD, - COMMAND_SEEK_BACK, - COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, - COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM) - - return State.Builder() - // Set which playback commands the player can handle - .setAvailableCommands(commands) - // Configure additional playback properties - .setPlayWhenReady(true, PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST) - .setCurrentMediaItemIndex(0) - .setContentPositionMs(0) - .setDeviceInfo(DeviceInfo.Builder(DeviceInfo.PLAYBACK_TYPE_LOCAL).build()) - .build() - } -} \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index d9d38936..d4e251f0 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -63,6 +63,12 @@ import org.scottishtecharmy.soundscape.locationprovider.DirectionProvider import org.scottishtecharmy.soundscape.locationprovider.LocationProvider import org.scottishtecharmy.soundscape.locationprovider.StaticLocationProvider import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.mediacontrol.OriginalMediaControls +import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeDummyMediaPlayer +import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeMediaSessionCallback +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommand +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandManager +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import org.scottishtecharmy.soundscape.utils.Analytics import org.scottishtecharmy.soundscape.utils.getCurrentLocale import kotlin.time.Duration @@ -158,7 +164,12 @@ class SoundscapeService : MediaSessionService() { // Media control button code private var mediaSession: MediaSession? = null - private val mediaPlayer = SoundscapeDummyMediaPlayer() + + // TODO: Pick what the media controls control + //private val mediaControlsTarget = VoiceCommandMediaControls(this) + private val mediaControlsTarget = OriginalMediaControls(this) + + private val mediaPlayer = SoundscapeDummyMediaPlayer(mediaControlsTarget) var running: Boolean = false var started: Boolean = false @@ -204,11 +215,6 @@ class SoundscapeService : MediaSessionService() { locationProvider.start(this) directionProvider.start(audioEngine, locationProvider) -// val configLocale = getCurrentLocale() -// val configuration = Configuration(applicationContext.resources.configuration) -// configuration.setLocale(configLocale) -// localizedContext = applicationContext.createConfigurationContext(configuration) -// if (::voiceCommandManager.isInitialized) voiceCommandManager.updateContext(localizedContext) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) } @@ -286,19 +292,12 @@ class SoundscapeService : MediaSessionService() { voiceCommandManager = VoiceCommandManager( context = this, onCommand = ::executeVoiceCommand, - onError = { - if (requestAudioFocus()) { - audioEngine.createEarcon( - NativeAudioEngine.EARCON_CALLOUTS_OFF, - AudioType.STANDARD - ) - audioEngine.createTextToSpeech("I'm sorry I didn't understand", AudioType.STANDARD) - } - } + onError = { } ) mediaSession = MediaSession.Builder(this, mediaPlayer) - .setCallback(SoundscapeMediaSessionCallback(this)) + .setId("org.scottishtecharmy.soundscape") + .setCallback(SoundscapeMediaSessionCallback(mediaControlsTarget)) .build() } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt new file mode 100644 index 00000000..3ab17a38 --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt @@ -0,0 +1,62 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import org.scottishtecharmy.soundscape.services.SoundscapeService + +// An interface that encapsulates the various media control buttons +interface MediaControlTarget { + + fun onPlayPause() : Boolean + fun onNext() : Boolean + fun onPrevious() : Boolean +} + +// This is how the iOS Soundscape media controls behaved +class OriginalMediaControls(val service: SoundscapeService) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + service.routeMute() + return true + } + + override fun onNext() : Boolean { + if (!service.routeSkipNext()) { + // If there's no route playing, callout My Location. + service.myLocation() + } + return true + } + + override fun onPrevious() : Boolean { + if(!service.routeSkipPrevious()) { + // If there's no route playing, callout around me + service.whatsAroundMe() + } + return true + } +} + +// This is how the voice command media controls work +class VoiceCommandMediaControls(val service: SoundscapeService) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + service.triggerVoiceCommand() + return true + } + override fun onNext() : Boolean { return false } + override fun onPrevious() : Boolean { return false } +} + +// This is how the menu navigation media controls work +class MenuMediaControls(val service: SoundscapeService) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + return true + } + override fun onNext() : Boolean { + return true + } + override fun onPrevious() : Boolean { + return true + } +} + diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt new file mode 100644 index 00000000..ac3b09fc --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt @@ -0,0 +1,69 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import android.os.Looper +import androidx.annotation.OptIn +import androidx.media3.common.DeviceInfo +import androidx.media3.common.MediaItem +import androidx.media3.common.MediaMetadata +import androidx.media3.common.Player +import androidx.media3.common.SimpleBasePlayer +import androidx.media3.common.util.UnstableApi +import com.google.common.util.concurrent.Futures +import com.google.common.util.concurrent.ListenableFuture + +@OptIn(UnstableApi::class) +class SoundscapeDummyMediaPlayer( + private val mediaControlTarget: MediaControlTarget +) : SimpleBasePlayer(Looper.getMainLooper()) { + + override fun getState(): State { + + val commands = Player.Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_STOP, + COMMAND_SEEK_TO_NEXT, + COMMAND_SEEK_TO_PREVIOUS, + COMMAND_SEEK_FORWARD, + COMMAND_SEEK_BACK, + COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, + COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM) + .build() + + val playlist = listOf( + MediaItemData.Builder("soundscape_item") + .setMediaItem( + MediaItem.Builder() + .setMediaId("soundscape_item") + .setMediaMetadata( + MediaMetadata.Builder() + .setTitle("Soundscape") + .setArtist("Scottish Tech Army") + .build() + ) + .build() + ) + .build() + ) + + return State.Builder() + // Set which playback commands the player can handle + .setAvailableCommands(commands) + // Configure additional playback properties + .setPlayWhenReady(true, PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST) + .setPlaylist(playlist) + .setCurrentMediaItemIndex(0) + .setContentPositionMs(0) + .setDeviceInfo(DeviceInfo.Builder(DeviceInfo.PLAYBACK_TYPE_LOCAL).build()) + .setPlaybackState(STATE_READY) + .build() + } + + override fun handleSetPlayWhenReady(playWhenReady: Boolean): ListenableFuture<*> { + mediaControlTarget.onPlayPause() + return Futures.immediateVoidFuture() + } + + override fun handleStop(): ListenableFuture<*> = + Futures.immediateVoidFuture() +} \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt similarity index 69% rename from app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt rename to app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt index b3f97674..dcc921f1 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt @@ -1,4 +1,4 @@ -package org.scottishtecharmy.soundscape.services +package org.scottishtecharmy.soundscape.services.mediacontrol import android.content.Intent import android.util.Log @@ -12,7 +12,9 @@ import androidx.media3.session.MediaSession /** This callback class handles media button events generated by bluetooth headphones etc. * These are then mapped to specific Soundscape features. */ -class SoundscapeMediaSessionCallback(val service : SoundscapeService): +class SoundscapeMediaSessionCallback( + val mediaControlsTarget: MediaControlTarget +): MediaSession.Callback { // Configure commands available to the controller in onConnect() @OptIn(UnstableApi::class) @@ -52,42 +54,29 @@ class SoundscapeMediaSessionCallback(val service : SoundscapeService): keyEvent?.let { event -> val decodedKey = when(event.keyCode) { - KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { service.triggerVoiceCommand(); "Play/Pause" } -// KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { -// // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. -// service.routeMute() -// "Play/Pause" -// } + KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { + // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. + mediaControlsTarget.onPlayPause() + "Play/Pause" + } KeyEvent.KEYCODE_MEDIA_SKIP_FORWARD -> { // ⏩ Skip Forward - if(!service.routeSkipNext()) { - // If there's no route playing, toggle auto callouts. - service.toggleAutoCallouts() - } + mediaControlsTarget.onNext() "Skip forward" } KeyEvent.KEYCODE_MEDIA_NEXT -> { // ⏭ Next - if(!service.routeSkipNext()) { - // If there's no route playing, callout My Location. - service.myLocation() - } + mediaControlsTarget.onNext() "Next" } KeyEvent.KEYCODE_MEDIA_PREVIOUS -> { // ⏮ Previous - if(!service.routeSkipPrevious()) { - // TODO: : Repeat last callout. - } - "Previous" + mediaControlsTarget.onPrevious() } KeyEvent.KEYCODE_MEDIA_SKIP_BACKWARD -> { // ⏪ Skip Backward - if(!service.routeSkipPrevious()) { - // If there's no route playing, callout around me - service.whatsAroundMe() - } + mediaControlsTarget.onPrevious() "Skip backward" } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt similarity index 86% rename from app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt rename to app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index e52813bf..b861796a 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -1,4 +1,4 @@ -package org.scottishtecharmy.soundscape.services +package org.scottishtecharmy.soundscape.services.mediacontrol import android.content.Context import android.content.Intent @@ -44,14 +44,22 @@ class VoiceCommandManager( // Must be called on the main thread (satisfied: service is on main thread) fun startListening() { if (_state.value is VoiceCommandState.Listening) return - if (!SpeechRecognizer.isRecognitionAvailable(context)) { onError(); return } - destroyRecognizer() - speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) - speechRecognizer?.setRecognitionListener(listener) + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + println("Recognition is unavailable") + onError() + return + } + if(speechRecognizer == null) { + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) + speechRecognizer?.setRecognitionListener(listener) + } val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + // TODO: We need to query the API to find out which Locales are supported and use one of + // those. For example, we might want to use es_ES even if we're in another country. + // https://medium.com/@andraz.pajtler/android-speech-to-text-the-missing-guide-part-1-824e2636c45a // Match recognizer language to the app's configured locale putExtra(RecognizerIntent.EXTRA_LANGUAGE, getCurrentLocale().toLanguageTag()) } @@ -80,6 +88,8 @@ class VoiceCommandManager( } override fun onError(error: Int) { + println("onError $error") + destroyRecognizer() _state.value = VoiceCommandState.Error onError() } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt index 06155612..af550eea 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt @@ -20,7 +20,7 @@ import org.scottishtecharmy.soundscape.SoundscapeServiceConnection import org.scottishtecharmy.soundscape.audio.AudioTour import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription -import org.scottishtecharmy.soundscape.services.VoiceCommandState +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import javax.inject.Inject @HiltViewModel From 1ad7b9e85b5fe402de5623ce4a7359a542ff84cd Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Mon, 23 Feb 2026 12:57:25 +0000 Subject: [PATCH 06/18] Initial audio menu implementation This relates to https://sta2020.atlassian.net/browse/SA-285 and implements the start of an Audio Menu controlled via media control buttons. This allows app control without using the phone UI. --- .../soundscape/geoengine/GeoEngine.kt | 2 +- .../soundscape/services/AudioMenu.kt | 308 ++++++++++++++++++ .../soundscape/services/SoundscapeService.kt | 37 ++- .../mediacontrol/MediaControlTarget.kt | 6 +- .../SoundscapeDummyMediaPlayer.kt | 4 +- .../SoundscapeMediaSessionCallback.kt | 18 +- app/src/main/res/values/strings.xml | 15 + 7 files changed, 371 insertions(+), 19 deletions(-) create mode 100644 app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt index 13985515..f8ba8755 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt @@ -438,7 +438,7 @@ class GeoEngine { // So long as the AudioEngine is not already busy, run any auto callouts that we // need. Auto Callouts use the direction of travel if there is one, otherwise // falling back to use the phone direction. - if((!soundscapeService.isAudioEngineBusy() || streetPreview.running) && !autoCalloutDisabled) { + if((!soundscapeService.isAudioEngineBusy() || streetPreview.running) && !autoCalloutDisabled && !soundscapeService.menuActive) { val callout = autoCallout.updateLocation( getCurrentUserGeometry(UserGeometry.HeadingMode.CourseAuto), diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt new file mode 100644 index 00000000..6451afe8 --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt @@ -0,0 +1,308 @@ +package org.scottishtecharmy.soundscape.services + +import android.content.Context +import android.content.res.Configuration +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.scottishtecharmy.soundscape.R +import org.scottishtecharmy.soundscape.audio.AudioType +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase +import org.scottishtecharmy.soundscape.utils.getCurrentLocale + +/** + * AudioMenu provides a hierarchical, navigable audio menu controlled by media buttons. + * + * - NEXT : advance to next item at the current level (wraps at end) + * - PREVIOUS: go back one item (wraps at end) + * - SELECT : enter a sub-menu, or execute a leaf action + * + * There is no inactivity timeout — the current position is remembered until changed. + */ +class AudioMenu( + private val service: SoundscapeService, + private val application: Context) { + + // ── Menu item types ─────────────────────────────────────────────────────── + + sealed class MenuItem { + abstract val label: String + + /** A leaf node that executes an action when selected. */ + data class Action( + override val label: String, + val action: () -> Unit + ) : MenuItem() + + /** A sub-menu with a fixed list of children. */ + data class Submenu( + override val label: String, + val children: List + ) : MenuItem() + + /** + * A sub-menu whose children are loaded lazily when the user enters it. + * Useful for content that may change at runtime (e.g. saved routes). + */ + data class DynamicSubmenu( + override val label: String, + val childrenProvider: suspend () -> List + ) : MenuItem() + } + + // ── Navigation state ────────────────────────────────────────────────────── + + private data class MenuLevel(val items: List, var currentIndex: Int) + + /** Stack of levels; the root is at index 0, deeper sub-menus higher up. */ + private val menuStack = ArrayDeque() + + private val scope = CoroutineScope(Dispatchers.Default) + + /** Cancels pending re-enable of auto callouts and restarts the 10-second countdown. */ + private var suppressionJob: Job? = null + + val localizedContext: Context + init { + val configLocale = getCurrentLocale() + val configuration = Configuration(application.applicationContext.resources.configuration) + configuration.setLocale(configLocale) + localizedContext = application.applicationContext.createConfigurationContext(configuration) + menuStack.addLast(MenuLevel(buildRootMenu(), 0)) + } + + // ── Public navigation API ───────────────────────────────────────────────── + + fun next() { + onMenuInteraction() + val label = synchronized(this) { + val level = menuStack.last() + level.currentIndex = (level.currentIndex + 1) % level.items.size + level.items[level.currentIndex].label + } + speak(label) + } + + fun previous() { + onMenuInteraction() + val label = synchronized(this) { + val level = menuStack.last() + level.currentIndex = + if(level.currentIndex == 0) + level.items.size - 1 + else + level.currentIndex - 1 + level.items[level.currentIndex].label + } + speak(label) + } + + fun select() { + onMenuInteraction() + val item = synchronized(this) { menuStack.last().let { it.items[it.currentIndex] } } + when (item) { + is MenuItem.Action -> { + if (!service.requestAudioFocus()) { + Log.w(TAG, "select: could not get audio focus") + return + } + service.audioEngine.clearTextToSpeechQueue() + item.action() + } + is MenuItem.Submenu -> { + val firstLabel = synchronized(this) { + menuStack.addLast(MenuLevel(item.children, 0)) + item.children[0].label + } + speakWithEnterEarcon(firstLabel) + } + is MenuItem.DynamicSubmenu -> loadAndEnter(item) + } + } + + // ── Auto-callout suppression ────────────────────────────────────────────── + + /** + * Called on every menu interaction. Marks the menu as active (suppressing auto callouts) + * and resets the 10-second countdown after which auto callouts are re-enabled. + */ + private fun onMenuInteraction() { + service.menuActive = true + suppressionJob?.cancel() + suppressionJob = scope.launch { + delay(CALLOUT_SUPPRESS_TIMEOUT_MS) + service.menuActive = false + } + } + + // ── Main-menu escape ────────────────────────────────────────────────────── + + /** + * Returns a "Main Menu" Action item that, when selected, pops the entire + * stack back to the root and announces the first root-level item. + * Appended as the last child of every sub-menu. + */ + private fun mainMenuAction(): MenuItem.Action = + MenuItem.Action(localizedContext.getString(R.string.menu_main_menu)) { + resetToRoot() + } + + private fun resetToRoot() { + val firstRootLabel = synchronized(this) { + while (menuStack.size > 1) menuStack.removeLast() + menuStack.last().currentIndex = 0 + menuStack.last().items[0].label + } + if (!service.requestAudioFocus()) { + Log.w(TAG, "resetToRoot: could not get audio focus") + return + } + service.audioEngine.clearTextToSpeechQueue() + service.audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_EXIT, AudioType.STANDARD) + service.audioEngine.createTextToSpeech(firstRootLabel, AudioType.STANDARD) + } + + // ── Audio helpers ───────────────────────────────────────────────────────── + + /** Announce a label with no earcon — used for plain navigation. */ + private fun speak(label: String) { + if (!service.requestAudioFocus()) { + Log.w(TAG, "speak: could not get audio focus") + return + } + service.audioEngine.clearTextToSpeechQueue() + service.audioEngine.createTextToSpeech(label, AudioType.STANDARD) + } + + /** Announce a label preceded by the mode-enter earcon — used when descending into a sub-menu. */ + private fun speakWithEnterEarcon(label: String) { + if (!service.requestAudioFocus()) { + Log.w(TAG, "speakWithEnterEarcon: could not get audio focus") + return + } + service.audioEngine.clearTextToSpeechQueue() + service.audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_ENTER, AudioType.STANDARD) + service.audioEngine.createTextToSpeech(label, AudioType.STANDARD) + } + + private fun loadAndEnter(item: MenuItem.DynamicSubmenu) { + scope.launch { + val children = item.childrenProvider() + if (children.size <= 1) { + service.speakText(localizedContext.getString(R.string.menu_no_routes), AudioType.STANDARD) + } else { + val firstLabel = synchronized(this@AudioMenu) { + menuStack.addLast(MenuLevel(children, 0)) + children[0].label + } + speakWithEnterEarcon(firstLabel) + } + } + } + + private fun audioProfileAction(@androidx.annotation.StringRes id: Int, profile: String): MenuItem.Action { + val label = localizedContext.getString(id) + return MenuItem.Action(label) { + applyAudioProfile(profile) + service.speakText(label, AudioType.STANDARD) + } + } + + // ── Menu definition ─────────────────────────────────────────────────────── + + private fun buildRootMenu(): List = listOf( + + MenuItem.Submenu( + label = localizedContext.getString(R.string.callouts_panel_title), + children = listOf( + MenuItem.Action(localizedContext.getString(R.string.directions_my_location)) { + service.myLocation() + }, + MenuItem.Action(localizedContext.getString(R.string.help_orient_page_title)) { + service.whatsAroundMe() + }, + MenuItem.Action(localizedContext.getString(R.string.help_explore_page_title)) { + service.aheadOfMe() + }, + MenuItem.Action(localizedContext.getString(R.string.callouts_nearby_markers)) { + service.nearbyMarkers() + }, + mainMenuAction(), + ) + ), + + MenuItem.Submenu( + label = localizedContext.getString(R.string.menu_audio_profile), + children = listOf( + audioProfileAction(R.string.menu_profile_eating, "eating"), + audioProfileAction(R.string.menu_profile_shopping, "shopping"), + audioProfileAction(R.string.menu_profile_navigating, "navigating"), + audioProfileAction(R.string.menu_profile_roads_only, "roads_only"), + audioProfileAction(R.string.menu_profile_all, "all"), + mainMenuAction(), + ) + ), + + MenuItem.Submenu( + label = localizedContext.getString(R.string.menu_route), + children = listOf( + MenuItem.Action(localizedContext.getString(R.string.menu_route_next_waypoint)) { + service.routeSkipNext() + }, + MenuItem.Action(localizedContext.getString(R.string.menu_route_previous_waypoint)) { + service.routeSkipPrevious() + }, + MenuItem.Action(localizedContext.getString(R.string.menu_route_stop)) { + service.routeStop() + }, + mainMenuAction(), + ) + ), + + MenuItem.DynamicSubmenu( + label = localizedContext.getString(R.string.menu_start_route), + childrenProvider = { loadRouteMenuItems() } + ), + ) + + // ── Feature implementations ─────────────────────────────────────────────── + + private suspend fun loadRouteMenuItems(): List = + withContext(Dispatchers.IO) { + val db = MarkersAndRoutesDatabase.getMarkersInstance(service) + db.routeDao().getAllRoutes().map { route -> + MenuItem.Action(route.name) { service.routeStart(route.routeId) } + } + mainMenuAction() + } + + /** + */ + private fun applyAudioProfile(profileName: String) { + when (profileName) { + "eating" -> {} + "shopping" -> {} + "navigating" -> {} + "roads_only" -> {} + "all" -> {} + } + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + fun destroy() { + suppressionJob?.cancel() + service.menuActive = false + scope.cancel() + } + + companion object { + private const val TAG = "AudioMenu" + private const val CALLOUT_SUPPRESS_TIMEOUT_MS = 8_000L + } +} diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index d4e251f0..e8da2093 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -63,11 +63,14 @@ import org.scottishtecharmy.soundscape.locationprovider.DirectionProvider import org.scottishtecharmy.soundscape.locationprovider.LocationProvider import org.scottishtecharmy.soundscape.locationprovider.StaticLocationProvider import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.mediacontrol.AudioMenuMediaControls +import org.scottishtecharmy.soundscape.services.mediacontrol.MediaControlTarget import org.scottishtecharmy.soundscape.services.mediacontrol.OriginalMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeDummyMediaPlayer import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeMediaSessionCallback import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommand import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandManager +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import org.scottishtecharmy.soundscape.utils.Analytics import org.scottishtecharmy.soundscape.utils.getCurrentLocale @@ -102,6 +105,12 @@ class SoundscapeService : MediaSessionService() { var audioEngine = NativeAudioEngine(this) private var audioBeacon: Long = 0 + // Audio menu (navigated via media buttons when no route is active) + var audioMenu : AudioMenu? = null + + /** True while the user is actively navigating the audio menu. Suppresses auto callouts. */ + var menuActive: Boolean = false + // Audio focus private lateinit var audioManager: AudioManager private var audioFocusRequest: AudioFocusRequest? = null @@ -165,11 +174,8 @@ class SoundscapeService : MediaSessionService() { // Media control button code private var mediaSession: MediaSession? = null - // TODO: Pick what the media controls control - //private val mediaControlsTarget = VoiceCommandMediaControls(this) - private val mediaControlsTarget = OriginalMediaControls(this) - - private val mediaPlayer = SoundscapeDummyMediaPlayer(mediaControlsTarget) + private var mediaControlsTarget : MediaControlTarget = OriginalMediaControls(this) + private val mediaPlayer = SoundscapeDummyMediaPlayer { mediaControlsTarget } var running: Boolean = false var started: Boolean = false @@ -275,9 +281,11 @@ class SoundscapeService : MediaSessionService() { audioEngine.initialize(applicationContext) audioManager = getSystemService(AUDIO_SERVICE) as AudioManager - + audioMenu = AudioMenu(this, application) routePlayer = RoutePlayer(this, applicationContext) + updateMediaControls(MediaControlsTarget.AUDIO_MENU) + if(hasPlayServices(this)) { locationProvider = GooglePlayLocationProvider(this) directionProvider = GooglePlayDirectionProvider(this) @@ -297,11 +305,25 @@ class SoundscapeService : MediaSessionService() { mediaSession = MediaSession.Builder(this, mediaPlayer) .setId("org.scottishtecharmy.soundscape") - .setCallback(SoundscapeMediaSessionCallback(mediaControlsTarget)) + .setCallback(SoundscapeMediaSessionCallback { mediaControlsTarget }) .build() } } + enum class MediaControlsTarget { + ORIGINAL, + VOICE_COMMAND, + AUDIO_MENU + } + fun updateMediaControls(target: MediaControlsTarget) { + mediaControlsTarget = when(target) { + MediaControlsTarget.ORIGINAL -> OriginalMediaControls(this) + MediaControlsTarget.VOICE_COMMAND -> VoiceCommandMediaControls(this) + MediaControlsTarget.AUDIO_MENU -> AudioMenuMediaControls(audioMenu) + } + + } + override fun onTaskRemoved(rootIntent: Intent?) { Log.d(TAG, "onTaskRemoved for service - ignoring, as we want to keep running") } @@ -318,6 +340,7 @@ class SoundscapeService : MediaSessionService() { super.onDestroy() Log.d(TAG, "onDestroy") + audioMenu?.destroy() audioEngine.destroyBeacon(audioBeacon) audioBeacon = 0 audioEngine.destroy() diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt index 3ab17a38..d3ba2827 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt @@ -1,5 +1,6 @@ package org.scottishtecharmy.soundscape.services.mediacontrol +import org.scottishtecharmy.soundscape.services.AudioMenu import org.scottishtecharmy.soundscape.services.SoundscapeService // An interface that encapsulates the various media control buttons @@ -47,15 +48,18 @@ class VoiceCommandMediaControls(val service: SoundscapeService) : MediaControlTa } // This is how the menu navigation media controls work -class MenuMediaControls(val service: SoundscapeService) : MediaControlTarget { +class AudioMenuMediaControls(val audioMenu: AudioMenu?) : MediaControlTarget { override fun onPlayPause() : Boolean { + audioMenu?.select() return true } override fun onNext() : Boolean { + audioMenu?.next() return true } override fun onPrevious() : Boolean { + audioMenu?.previous() return true } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt index ac3b09fc..5efd95b0 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt @@ -13,7 +13,7 @@ import com.google.common.util.concurrent.ListenableFuture @OptIn(UnstableApi::class) class SoundscapeDummyMediaPlayer( - private val mediaControlTarget: MediaControlTarget + private val getTarget: () -> MediaControlTarget ) : SimpleBasePlayer(Looper.getMainLooper()) { override fun getState(): State { @@ -60,7 +60,7 @@ class SoundscapeDummyMediaPlayer( } override fun handleSetPlayWhenReady(playWhenReady: Boolean): ListenableFuture<*> { - mediaControlTarget.onPlayPause() + getTarget().onPlayPause() return Futures.immediateVoidFuture() } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt index dcc921f1..41d4dd20 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt @@ -13,7 +13,7 @@ import androidx.media3.session.MediaSession * These are then mapped to specific Soundscape features. */ class SoundscapeMediaSessionCallback( - val mediaControlsTarget: MediaControlTarget + private val getTarget: () -> MediaControlTarget ): MediaSession.Callback { // Configure commands available to the controller in onConnect() @@ -49,34 +49,36 @@ class SoundscapeMediaSessionCallback( // KEYCODE_MEDIA_NEXT, though that may be specific to my phone. The only event actually // handled for now is KEYCODE_MEDIA_NEXT. - // The behaviour of the media buttons changes when a route is being played back. In that - // case the buttons map to next/previous waypoint and muting audio. + // TODO: + // The behaviour of the media buttons changes when a route is being played back. In that + // case the buttons map to next/previous waypoint and muting audio. This currently doesn't + // play nicely with the audioMenu, more work is required. keyEvent?.let { event -> val decodedKey = when(event.keyCode) { KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. - mediaControlsTarget.onPlayPause() + getTarget().onPlayPause() "Play/Pause" } KeyEvent.KEYCODE_MEDIA_SKIP_FORWARD -> { // ⏩ Skip Forward - mediaControlsTarget.onNext() + getTarget().onNext() "Skip forward" } KeyEvent.KEYCODE_MEDIA_NEXT -> { // ⏭ Next - mediaControlsTarget.onNext() + getTarget().onNext() "Next" } KeyEvent.KEYCODE_MEDIA_PREVIOUS -> { // ⏮ Previous - mediaControlsTarget.onPrevious() + getTarget().onPrevious() } KeyEvent.KEYCODE_MEDIA_SKIP_BACKWARD -> { // ⏪ Skip Backward - mediaControlsTarget.onPrevious() + getTarget().onPrevious() "Skip backward" } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 9d50306c..6f6503c7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -478,6 +478,21 @@ "Ahead\nof Me" "Hear about places in front of you" + +"Select audio profile" +"Eating" +"Shopping" +"Navigating" +"Roads only" +"No filtering" +"Route control" +"Next waypoint" +"Previous waypoint" +"Stop" +"Choose route to start" +"No routes saved" +"Top menu" + "Menu" From 2ebeff722339b02f0508f7392b1cb78bd4668f5c Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 27 Feb 2026 14:30:06 +0000 Subject: [PATCH 07/18] Add Settings to control which mode the Media Controls run in Media control buttons can be working in: Original mode - similar to on iOS. Relatively hard coded. Voice control - the play/pause button triggers voice control. Audio menu - a whole audio menu is navigable via the controls. --- .../soundscape/MainActivity.kt | 8 +++ .../screens/home/settings/Settings.kt | 52 +++++++++++++++++++ .../soundscape/services/SoundscapeService.kt | 11 ++-- app/src/main/res/values/strings.xml | 12 +++++ 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt index be53d10d..f8fde362 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt @@ -168,6 +168,12 @@ class MainActivity : AppCompatActivity() { hintsEnabled = preferences.getBoolean(HINTS_KEY, HINTS_DEFAULT) ) } + + MEDIA_CONTROLS_MODE_KEY -> { + val mode = preferences.getString(MEDIA_CONTROLS_MODE_KEY, MEDIA_CONTROLS_MODE_DEFAULT)!! + Log.e(TAG, "mediaControlsMode $mode") + soundscapeServiceConnection.soundscapeService?.updateMediaControls(mode) + } } } @@ -805,6 +811,8 @@ class MainActivity : AppCompatActivity() { const val GEOCODER_MODE_KEY = "GeocoderMode" const val LAST_SPLASH_RELEASE_DEFAULT = "" const val LAST_SPLASH_RELEASE_KEY = "LastNewRelease" + const val MEDIA_CONTROLS_MODE_DEFAULT = "Original" + const val MEDIA_CONTROLS_MODE_KEY = "MediaControlsMode" const val FIRST_LAUNCH_KEY = "FirstLaunch" const val AUDIO_TOUR_SHOWN_KEY = "AudioTourShown" diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt index 3893c4b3..78adb1fd 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt @@ -258,6 +258,17 @@ fun Settings( "Offline" ) + val mediaControlsDescriptions = listOf( + stringResource(R.string.settings_media_controls_original), + stringResource(R.string.settings_media_controls_voice_command), + stringResource(R.string.settings_media_controls_audio_menu), + ) + val mediaControlsValues = listOf( + "Original", + "VoiceControl", + "AudioMenu", + ) + if (showConfirmationDialog.value) { AlertDialog( onDismissRequest = { showConfirmationDialog.value = false }, @@ -683,6 +694,42 @@ fun Settings( } } + // Media control section + item(key = "header_media_control") { + ExpandableSectionHeader( + title = stringResource(R.string.menu_media_controls), + expanded = expandedSection.value == "media_controls", + onToggle = { expandedSection.value = if (expandedSection.value == "media_controls") null else "media_controls" }, + textColor = textColor + ) + } + if (expandedSection.value == "media_controls") { + listPreference( + key = MainActivity.MEDIA_CONTROLS_MODE_KEY, + defaultValue = MainActivity.MEDIA_CONTROLS_MODE_DEFAULT, + values = mediaControlsValues, + modifier = expandedSectionModifier, + title = { + SettingDetails( + R.string.settings_section_media_controls, + R.string.settings_section_media_controls_description, + textColor + ) + }, + item = { value, currentValue, onClick -> + ListPreferenceItem(mediaControlsDescriptions[mediaControlsValues.indexOf(value)], value, currentValue, onClick, mediaControlsValues.indexOf(value), mediaControlsValues.size) + }, + summary = { + Text( + text = mediaControlsDescriptions[mediaControlsValues.indexOf(it)], + color = textColor, + style = MaterialTheme.typography.bodyLarge + ) + }, + ) + } + + // Debug Section item(key = "header_debug") { ExpandableSectionHeader( @@ -800,6 +847,11 @@ fun SettingsPreviewLanguage() { } @Preview @Composable +fun SettingsPreviewMediaControls() { + SettingsPreview("media_controls") +} +@Preview +@Composable fun SettingsPreviewDebug() { SettingsPreview("debug") } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index e8da2093..f4bd4291 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -284,8 +284,6 @@ class SoundscapeService : MediaSessionService() { audioMenu = AudioMenu(this, application) routePlayer = RoutePlayer(this, applicationContext) - updateMediaControls(MediaControlsTarget.AUDIO_MENU) - if(hasPlayServices(this)) { locationProvider = GooglePlayLocationProvider(this) directionProvider = GooglePlayDirectionProvider(this) @@ -315,11 +313,12 @@ class SoundscapeService : MediaSessionService() { VOICE_COMMAND, AUDIO_MENU } - fun updateMediaControls(target: MediaControlsTarget) { + fun updateMediaControls(target: String) { mediaControlsTarget = when(target) { - MediaControlsTarget.ORIGINAL -> OriginalMediaControls(this) - MediaControlsTarget.VOICE_COMMAND -> VoiceCommandMediaControls(this) - MediaControlsTarget.AUDIO_MENU -> AudioMenuMediaControls(audioMenu) + "Original" -> OriginalMediaControls(this) + "VoiceControl" -> VoiceCommandMediaControls(this) + "AudioMenu" -> AudioMenuMediaControls(audioMenu) + else -> OriginalMediaControls(this) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 6f6503c7..56cf5025 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -854,6 +854,18 @@ "Language and Region " "Accessibility" + +"Media Controls" + +"Media controls mode" + +"The media control buttons on your headphones, or remote can be configured to work in different ways depending on this option." + +"Original mode" + +"Voice command" + +"Audio menu" "Light or Dark Theme" From b293e66c0cc85409eb8c1001ac4964c2a42d48ab Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 27 Feb 2026 14:49:02 +0000 Subject: [PATCH 08/18] Improve voice command help --- .../soundscape/services/SoundscapeService.kt | 14 +++++++++++++- app/src/main/res/values/strings.xml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index f4bd4291..9dbc106a 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -592,7 +592,19 @@ class SoundscapeService : MediaSessionService() { VoiceCommand.SKIP_PREVIOUS -> { routeSkipPrevious(); null } VoiceCommand.MUTE -> { routeMute(); null } VoiceCommand.STOP_ROUTE -> { routeStop(); null } - VoiceCommand.HELP -> ctx.getString(R.string.voice_cmd_help_response) + VoiceCommand.HELP -> { + val commandNames = listOf( + R.string.voice_cmd_my_location, + R.string.voice_cmd_around_me, + R.string.voice_cmd_ahead_of_me, + R.string.voice_cmd_nearby_markers, + R.string.voice_cmd_skip_previous, + R.string.voice_cmd_skip_next, + R.string.voice_cmd_mute, + R.string.voice_cmd_stop_route + ).map { ctx.getString(it).substringBefore(',') } + ctx.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(", ") + } VoiceCommand.UNKNOWN -> { audioEngine.createEarcon(NativeAudioEngine.EARCON_CALLOUTS_OFF, AudioType.STANDARD) "I'm sorry I didn't understand" diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 56cf5025..d0172843 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2616,7 +2616,7 @@ Please report any problems that you have, however small they may be via the *Con stop route,stop help,what can I say,commands,what commands -Available commands: +Available voice commands: Listening… From 98ca6ebd3e45b3d91da5a201a65d3da8e185b0f0 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Sat, 28 Feb 2026 08:26:39 +0000 Subject: [PATCH 09/18] Move callout hold off into service to share with media controls The callout holdoff initially was for the audio menu, but is useful for voice control too. --- .../soundscape/services/AudioMenu.kt | 28 ++----------------- .../soundscape/services/SoundscapeService.kt | 23 +++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt index 6451afe8..8128af5f 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt @@ -5,9 +5,7 @@ import android.content.res.Configuration import android.util.Log import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.scottishtecharmy.soundscape.R @@ -65,9 +63,6 @@ class AudioMenu( private val scope = CoroutineScope(Dispatchers.Default) - /** Cancels pending re-enable of auto callouts and restarts the 10-second countdown. */ - private var suppressionJob: Job? = null - val localizedContext: Context init { val configLocale = getCurrentLocale() @@ -80,7 +75,7 @@ class AudioMenu( // ── Public navigation API ───────────────────────────────────────────────── fun next() { - onMenuInteraction() + service.callbackHoldOff() val label = synchronized(this) { val level = menuStack.last() level.currentIndex = (level.currentIndex + 1) % level.items.size @@ -90,7 +85,7 @@ class AudioMenu( } fun previous() { - onMenuInteraction() + service.callbackHoldOff() val label = synchronized(this) { val level = menuStack.last() level.currentIndex = @@ -104,7 +99,7 @@ class AudioMenu( } fun select() { - onMenuInteraction() + service.callbackHoldOff() val item = synchronized(this) { menuStack.last().let { it.items[it.currentIndex] } } when (item) { is MenuItem.Action -> { @@ -126,21 +121,6 @@ class AudioMenu( } } - // ── Auto-callout suppression ────────────────────────────────────────────── - - /** - * Called on every menu interaction. Marks the menu as active (suppressing auto callouts) - * and resets the 10-second countdown after which auto callouts are re-enabled. - */ - private fun onMenuInteraction() { - service.menuActive = true - suppressionJob?.cancel() - suppressionJob = scope.launch { - delay(CALLOUT_SUPPRESS_TIMEOUT_MS) - service.menuActive = false - } - } - // ── Main-menu escape ────────────────────────────────────────────────────── /** @@ -296,13 +276,11 @@ class AudioMenu( // ── Lifecycle ───────────────────────────────────────────────────────────── fun destroy() { - suppressionJob?.cancel() service.menuActive = false scope.cancel() } companion object { private const val TAG = "AudioMenu" - private const val CALLOUT_SUPPRESS_TIMEOUT_MS = 8_000L } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 9dbc106a..9e0ecc9b 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -327,6 +327,8 @@ class SoundscapeService : MediaSessionService() { Log.d(TAG, "onTaskRemoved for service - ignoring, as we want to keep running") } override fun onDestroy() { + suppressionJob?.cancel() + // If _mediaSession is not null, run the following block mediaSession?.run { // Release the player @@ -561,6 +563,9 @@ class SoundscapeService : MediaSessionService() { return } + // Stop callbacks whilst we handle voice commands + callbackHoldOff() + val ctx = if (::localizedContext.isInitialized) localizedContext else this // Clear the text queue @@ -811,12 +816,30 @@ class SoundscapeService : MediaSessionService() { audioFocusGained = false } + /** + * Called on every menu interaction. Marks the menu as active (suppressing auto callouts) + * and resets the 10-second countdown after which auto callouts are re-enabled. + */ + /** Cancels pending re-enable of auto callouts and restarts the 10-second countdown. */ + private var suppressionJob: Job? = null + private val scope = CoroutineScope(Dispatchers.Default) + fun callbackHoldOff() { + menuActive = true + suppressionJob?.cancel() + suppressionJob = scope.launch { + delay(CALLOUT_SUPPRESS_TIMEOUT_MS) + menuActive = false + } + } + companion object { private const val TAG = "SoundscapeService" // Secondary "service" every n seconds private val TICKER_PERIOD_SECONDS = 3600.seconds + private const val CALLOUT_SUPPRESS_TIMEOUT_MS = 8_000L + private const val CHANNEL_ID = "SoundscapeService_channel_01" private const val NOTIFICATION_CHANNEL_NAME = "Soundscape_SoundscapeService" private const val NOTIFICATION_ID = 100000 From bca37250b9803733bda1e3fa36b2d43e7235bc99 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Sat, 28 Feb 2026 10:29:12 +0000 Subject: [PATCH 10/18] Match voice command and audio menu code with document describing interface Voice commands and the audio menu can now be used to start audio beacons at markers, and route playback. --- .../soundscape/services/SoundscapeService.kt | 166 +++++++++++++----- .../services/{ => mediacontrol}/AudioMenu.kt | 39 ++-- .../mediacontrol/MediaControlTarget.kt | 1 - .../mediacontrol/VoiceCommandManager.kt | 112 ++++++++---- app/src/main/res/values/strings.xml | 38 ++-- 5 files changed, 253 insertions(+), 103 deletions(-) rename app/src/main/java/org/scottishtecharmy/soundscape/services/{ => mediacontrol}/AudioMenu.kt (90%) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 9e0ecc9b..f1c9e29f 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -29,6 +29,7 @@ import androidx.core.content.ContextCompat import androidx.media3.common.util.UnstableApi import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService +import androidx.preference.PreferenceManager import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -42,6 +43,8 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.scottishtecharmy.soundscape.MainActivity +import org.scottishtecharmy.soundscape.MainActivity.Companion.MEDIA_CONTROLS_MODE_DEFAULT +import org.scottishtecharmy.soundscape.MainActivity.Companion.MEDIA_CONTROLS_MODE_KEY import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.AudioType import org.scottishtecharmy.soundscape.audio.NativeAudioEngine @@ -63,16 +66,17 @@ import org.scottishtecharmy.soundscape.locationprovider.DirectionProvider import org.scottishtecharmy.soundscape.locationprovider.LocationProvider import org.scottishtecharmy.soundscape.locationprovider.StaticLocationProvider import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.mediacontrol.AudioMenu import org.scottishtecharmy.soundscape.services.mediacontrol.AudioMenuMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.MediaControlTarget import org.scottishtecharmy.soundscape.services.mediacontrol.OriginalMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeDummyMediaPlayer import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeMediaSessionCallback -import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommand import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandManager import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import org.scottishtecharmy.soundscape.utils.Analytics +import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds @@ -295,9 +299,13 @@ class SoundscapeService : MediaSessionService() { // create new RealmDB or open existing startRealms(applicationContext) + // Update the media controls mode + val sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) + val mode = sharedPreferences.getString(MEDIA_CONTROLS_MODE_KEY, MEDIA_CONTROLS_MODE_DEFAULT)!! + updateMediaControls(mode) + voiceCommandManager = VoiceCommandManager( - context = this, - onCommand = ::executeVoiceCommand, + service = this, onError = { } ) @@ -308,11 +316,6 @@ class SoundscapeService : MediaSessionService() { } } - enum class MediaControlsTarget { - ORIGINAL, - VOICE_COMMAND, - AUDIO_MENU - } fun updateMediaControls(target: String) { mediaControlsTarget = when(target) { "Original" -> OriginalMediaControls(this) @@ -586,40 +589,6 @@ class SoundscapeService : MediaSessionService() { } } - fun executeVoiceCommand(command: VoiceCommand) { - val ctx = if (::localizedContext.isInitialized) localizedContext else this - val label = when (command) { - VoiceCommand.MY_LOCATION -> { myLocation(); null } - VoiceCommand.AROUND_ME -> { whatsAroundMe(); null } - VoiceCommand.AHEAD_OF_ME -> { aheadOfMe(); null } - VoiceCommand.NEARBY_MARKERS -> { nearbyMarkers(); null } - VoiceCommand.SKIP_NEXT -> { routeSkipNext(); null } - VoiceCommand.SKIP_PREVIOUS -> { routeSkipPrevious(); null } - VoiceCommand.MUTE -> { routeMute(); null } - VoiceCommand.STOP_ROUTE -> { routeStop(); null } - VoiceCommand.HELP -> { - val commandNames = listOf( - R.string.voice_cmd_my_location, - R.string.voice_cmd_around_me, - R.string.voice_cmd_ahead_of_me, - R.string.voice_cmd_nearby_markers, - R.string.voice_cmd_skip_previous, - R.string.voice_cmd_skip_next, - R.string.voice_cmd_mute, - R.string.voice_cmd_stop_route - ).map { ctx.getString(it).substringBefore(',') } - ctx.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(", ") - } - VoiceCommand.UNKNOWN -> { - audioEngine.createEarcon(NativeAudioEngine.EARCON_CALLOUTS_OFF, AudioType.STANDARD) - "I'm sorry I didn't understand" - } - } - if (requestAudioFocus()) { - label?.let { audioEngine.createTextToSpeech(it, AudioType.STANDARD) } - } - } - suspend fun searchResult(searchString: String): List? { return geoEngine.searchResult(searchString) } @@ -657,6 +626,119 @@ class SoundscapeService : MediaSessionService() { } return false } + fun routeListRoutes() { + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val routes = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllRoutes() + if (requestAudioFocus()) { + if (routes.isEmpty()) { + audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_no_routes), AudioType.STANDARD) + } else { + val names = routes.joinToString(", ") { it.name } + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_routes_list) + names, + AudioType.STANDARD + ) + } + } + } + } + + fun routeStartByName(name: String) { + if (name.isEmpty()) { + routeListRoutes() + return + } + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val routes = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllRoutes() + val nameLower = name.lowercase() + var bestId = -1L + var bestScore = Double.MAX_VALUE + for (route in routes) { + val score = nameLower.fuzzyCompare(route.name.lowercase(), true) + if (score < 0.4 && score < bestScore) { + bestScore = score + bestId = route.routeId + } + } + if (bestId != -1L) { + val routeName = routes.first { it.routeId == bestId }.name + if (requestAudioFocus()) { + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_starting_route).format(routeName), + AudioType.STANDARD + ) + } + routeStart(bestId) + } else { + if (requestAudioFocus()) { + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_route_not_found).format(name), + AudioType.STANDARD + ) + } + } + } + } + + fun routeListMarkers() { + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val markers = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllMarkers() + if (requestAudioFocus()) { + if (markers.isEmpty()) { + audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_no_markers), AudioType.STANDARD) + } else { + val names = markers.joinToString(", ") { it.name } + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_markers_list) + names, + AudioType.STANDARD + ) + } + } + } + } + + fun markerStartByName(name: String) { + if (name.isEmpty()) { + routeListMarkers() + return + } + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val markers = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllMarkers() + val nameLower = name.lowercase() + var bestId = -1L + var bestScore = Double.MAX_VALUE + for (marker in markers) { + val score = nameLower.fuzzyCompare(marker.name.lowercase(), true) + if (score < 0.4 && score < bestScore) { + bestScore = score + bestId = marker.markerId + } + } + if (bestId != -1L) { + val marker = markers.first { it.markerId == bestId } + if (requestAudioFocus()) { + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name), + AudioType.STANDARD + ) + } + val location = LngLatAlt(marker.longitude, marker.latitude) + startBeacon(location, marker.name) + } else { + if (requestAudioFocus()) { + audioEngine.createTextToSpeech( + ctx.getString(R.string.voice_cmd_marker_not_found).format(name), + AudioType.STANDARD + ) + } + } + } + } + /** * isAudioEngineBusy returns true if there is more than one entry in the * audio engine queue. The queue consists of earcons and text-to-speech. diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt similarity index 90% rename from app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt rename to app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt index 8128af5f..2aac7fde 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/AudioMenu.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt @@ -1,4 +1,4 @@ -package org.scottishtecharmy.soundscape.services +package org.scottishtecharmy.soundscape.services.mediacontrol import android.content.Context import android.content.res.Configuration @@ -12,6 +12,8 @@ import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.AudioType import org.scottishtecharmy.soundscape.audio.NativeAudioEngine import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase +import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.services.SoundscapeService import org.scottishtecharmy.soundscape.utils.getCurrentLocale /** @@ -217,18 +219,6 @@ class AudioMenu( ) ), - MenuItem.Submenu( - label = localizedContext.getString(R.string.menu_audio_profile), - children = listOf( - audioProfileAction(R.string.menu_profile_eating, "eating"), - audioProfileAction(R.string.menu_profile_shopping, "shopping"), - audioProfileAction(R.string.menu_profile_navigating, "navigating"), - audioProfileAction(R.string.menu_profile_roads_only, "roads_only"), - audioProfileAction(R.string.menu_profile_all, "all"), - mainMenuAction(), - ) - ), - MenuItem.Submenu( label = localizedContext.getString(R.string.menu_route), children = listOf( @@ -238,7 +228,10 @@ class AudioMenu( MenuItem.Action(localizedContext.getString(R.string.menu_route_previous_waypoint)) { service.routeSkipPrevious() }, - MenuItem.Action(localizedContext.getString(R.string.menu_route_stop)) { + MenuItem.Action(localizedContext.getString(R.string.beacon_action_mute_beacon)) { + service.routeMute() + }, + MenuItem.Action(localizedContext.getString(R.string.route_detail_action_stop_route)) { service.routeStop() }, mainMenuAction(), @@ -246,9 +239,14 @@ class AudioMenu( ), MenuItem.DynamicSubmenu( - label = localizedContext.getString(R.string.menu_start_route), + label = localizedContext.getString(R.string.route_detail_action_start_route), childrenProvider = { loadRouteMenuItems() } ), + + MenuItem.DynamicSubmenu( + label = localizedContext.getString(R.string.location_detail_action_beacon), + childrenProvider = { loadMarkerMenuItems() } + ), ) // ── Feature implementations ─────────────────────────────────────────────── @@ -261,6 +259,17 @@ class AudioMenu( } + mainMenuAction() } + private suspend fun loadMarkerMenuItems(): List = + withContext(Dispatchers.IO) { + val db = MarkersAndRoutesDatabase.getMarkersInstance(service) + db.routeDao().getAllMarkers().map { marker -> + MenuItem.Action(marker.name) { + val location = LngLatAlt(marker.longitude, marker.latitude) + service.startBeacon(location, marker.name) + } + } + mainMenuAction() + } + /** */ private fun applyAudioProfile(profileName: String) { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt index d3ba2827..77e36181 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt @@ -1,6 +1,5 @@ package org.scottishtecharmy.soundscape.services.mediacontrol -import org.scottishtecharmy.soundscape.services.AudioMenu import org.scottishtecharmy.soundscape.services.SoundscapeService // An interface that encapsulates the various media control buttons diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index b861796a..2ad2b0fb 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -10,6 +10,10 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import org.scottishtecharmy.soundscape.R +import org.scottishtecharmy.soundscape.audio.AudioType +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.services.SoundscapeService +import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale sealed class VoiceCommandState { @@ -18,20 +22,12 @@ sealed class VoiceCommandState { object Error : VoiceCommandState() } -enum class VoiceCommand { - MY_LOCATION, AROUND_ME, AHEAD_OF_ME, NEARBY_MARKERS, - SKIP_NEXT, SKIP_PREVIOUS, MUTE, STOP_ROUTE, HELP, UNKNOWN -} - class VoiceCommandManager( - // Mutable so SoundscapeService can push localizedContext once it's created. - // The service context is used for SpeechRecognizer binding; - // the localized context is used for string-resource keyword lookups. - private var context: Context, - private val onCommand: (VoiceCommand) -> Unit, + private val service: SoundscapeService, private val onError: () -> Unit ) { + private var context: Context = service private var speechRecognizer: SpeechRecognizer? = null private val _state = MutableStateFlow(VoiceCommandState.Idle) val state: StateFlow = _state.asStateFlow() @@ -49,12 +45,15 @@ class VoiceCommandManager( onError() return } - if(speechRecognizer == null) { + if (speechRecognizer == null) { speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) speechRecognizer?.setRecognitionListener(listener) } val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra( + RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM + ) putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) // TODO: We need to query the API to find out which Locales are supported and use one of @@ -84,7 +83,8 @@ class VoiceCommandManager( _state.value = VoiceCommandState.Idle val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) matches?.forEach { println("speech: $it") } - onCommand(parseCommand(matches?.firstOrNull() ?: "")) + if (matches != null) + handleSpeech(matches) } override fun onError(error: Int) { @@ -102,25 +102,73 @@ class VoiceCommandManager( override fun onEvent(eventType: Int, params: Bundle?) {} } - private fun parseCommand(text: String): VoiceCommand { - val t = text.lowercase() - - // Match against localized keyword lists stored in string resources. - // Each resource is a comma-separated list of phrases a user might say. - fun matches(resId: Int) = - context.getString(resId).split(",").any { t.contains(it.trim()) } - - return when { - matches(R.string.voice_cmd_my_location) -> VoiceCommand.MY_LOCATION - matches(R.string.voice_cmd_around_me) -> VoiceCommand.AROUND_ME - matches(R.string.voice_cmd_ahead_of_me) -> VoiceCommand.AHEAD_OF_ME - matches(R.string.voice_cmd_nearby_markers) -> VoiceCommand.NEARBY_MARKERS - matches(R.string.voice_cmd_skip_previous) -> VoiceCommand.SKIP_PREVIOUS - matches(R.string.voice_cmd_skip_next) -> VoiceCommand.SKIP_NEXT - matches(R.string.voice_cmd_mute) -> VoiceCommand.MUTE - matches(R.string.voice_cmd_stop_route) -> VoiceCommand.STOP_ROUTE - matches(R.string.voice_cmd_help) -> VoiceCommand.HELP - else -> VoiceCommand.UNKNOWN + data class VoiceCommand(val stringId: Int, val action: (arg: String) -> Unit) + + val commands = arrayOf( + VoiceCommand(R.string.directions_my_location) { service.myLocation() }, + VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, + VoiceCommand(R.string.help_explore_page_title) { service.aheadOfMe() }, + VoiceCommand(R.string.callouts_nearby_markers) { service.nearbyMarkers() }, + VoiceCommand(R.string.route_detail_action_next) { service.routeSkipNext() }, + VoiceCommand(R.string.route_detail_action_previous) { service.routeSkipPrevious() }, + VoiceCommand(R.string.beacon_action_mute_beacon) { service.routeMute() }, + VoiceCommand(R.string.route_detail_action_stop_route) { service.routeStop() }, + VoiceCommand(R.string.voice_cmd_list_routes) { service.routeListRoutes() }, + VoiceCommand(R.string.route_detail_action_start_route) { + // TODO: We need a "fuzzy remove" here + val routeName = it.removePrefix(context.getString(R.string.route_detail_action_start_route).lowercase()).trim() + service.routeStartByName(routeName) + }, + VoiceCommand(R.string.voice_cmd_list_markers) { service.routeListMarkers() }, + VoiceCommand(R.string.voice_cmd_start_beacon_at_marker) { + // TODO: We need a "fuzzy remove" here + val markerName = it.removePrefix(context.getString(R.string.location_detail_action_beacon).lowercase()).trim() + service.markerStartByName(markerName) + }, + VoiceCommand(R.string.menu_help) { voiceHelp() }, + ) + + private fun handleSpeech(speech: ArrayList) { + + // TODO: Start by only looking at the very first string for a match. We should check the + // other strings too. + + // Find the best match to the speech. + val t = speech.first().lowercase() + + var minMatch = Double.MAX_VALUE + var bestMatch: VoiceCommand? = null + for (command in commands) { + val commandString = context.getString(command.stringId).lowercase() + val match = commandString.fuzzyCompare(t, true) + if ((match < 0.3) && (match < minMatch)) { + minMatch = match + bestMatch = command + } + } + if (bestMatch != null) { + println("Found command: ${context.getString(bestMatch.stringId)}") + bestMatch.action(t) + } else { + if (service.requestAudioFocus()) { + service.audioEngine.createEarcon( + NativeAudioEngine.EARCON_CALLOUTS_OFF, + AudioType.STANDARD + ) + service.audioEngine.createTextToSpeech( + context.getString(R.string.voice_cmd_not_recognized).format(t), + AudioType.STANDARD + ) + } + } + } + + private fun voiceHelp() { + val commandNames = commands.map { context.getString(it.stringId) } + val text = + context.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(", ") + if (service.requestAudioFocus()) { + service.audioEngine.createTextToSpeech(text, AudioType.STANDARD) } } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index d0172843..d74002a2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2603,20 +2603,32 @@ Please report any problems that you have, however small they may be via the *Con Continue Close the popup and continue with the tutorial - - -my location,where am i -"around me,what's around,around" -"ahead of me,what's ahead,ahead" -nearby markers,markers,nearby -skip next,next waypoint,next,skip -skip previous,previous waypoint,previous,go back -mute,silence -stop route,stop -help,what can I say,commands,what commands Available voice commands: - + Listening… + +"I'm sorry, I can't find the command %s" + +List routes + +Available routes are, + +List markers + +Available markers are, + +No routes have been saved + +No markers have been saved + +Route not found: %s + +Starting route %s + +Start audio beacon at %s + +Starting audio beacon at %s + +Marker not found: %s From fd223df4306d4e287d9e35edd0f3a885e8360fb2 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Sat, 28 Feb 2026 15:05:53 +0000 Subject: [PATCH 11/18] Tidy up speech output for audio menu and voice command into speak2dText Nothing very clever here, just a bit of refactoring. --- .../soundscape/services/SoundscapeService.kt | 89 ++++++++----------- .../services/mediacontrol/AudioMenu.kt | 45 ++-------- .../mediacontrol/VoiceCommandManager.kt | 22 ++--- app/src/main/res/values/strings.xml | 8 -- 4 files changed, 51 insertions(+), 113 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index f1c9e29f..065dd940 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -48,6 +48,9 @@ import org.scottishtecharmy.soundscape.MainActivity.Companion.MEDIA_CONTROLS_MOD import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.AudioType import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_CALLOUTS_ON +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_ENTER +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_EXIT import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase import org.scottishtecharmy.soundscape.geoengine.GeoEngine import org.scottishtecharmy.soundscape.geoengine.GridState @@ -571,11 +574,8 @@ class SoundscapeService : MediaSessionService() { val ctx = if (::localizedContext.isInitialized) localizedContext else this - // Clear the text queue - audioEngine.clearTextToSpeechQueue() - // Create the earcon and Listening... speech - audioEngine.createEarcon(NativeAudioEngine.EARCON_CALLOUTS_ON, AudioType.STANDARD) - audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_listening), AudioType.STANDARD) + // Inform the user that we're listening + speak2dText(ctx.getString(R.string.voice_cmd_listening), true, EARCON_CALLOUTS_ON) // Wait for the TTS to finish before opening the mic coroutineScope.launch { @@ -630,16 +630,11 @@ class SoundscapeService : MediaSessionService() { coroutineScope.launch { val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService val routes = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllRoutes() - if (requestAudioFocus()) { - if (routes.isEmpty()) { - audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_no_routes), AudioType.STANDARD) - } else { - val names = routes.joinToString(", ") { it.name } - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_routes_list) + names, - AudioType.STANDARD - ) - } + if (routes.isEmpty()) + speak2dText(ctx.getString(R.string.voice_cmd_no_routes)) + else { + val names = routes.joinToString(", ") { it.name } + speak2dText(ctx.getString(R.string.voice_cmd_routes_list) + names) } } } @@ -664,20 +659,10 @@ class SoundscapeService : MediaSessionService() { } if (bestId != -1L) { val routeName = routes.first { it.routeId == bestId }.name - if (requestAudioFocus()) { - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_starting_route).format(routeName), - AudioType.STANDARD - ) - } + speak2dText(ctx.getString(R.string.voice_cmd_starting_route).format(routeName)) routeStart(bestId) } else { - if (requestAudioFocus()) { - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_route_not_found).format(name), - AudioType.STANDARD - ) - } + speak2dText(ctx.getString(R.string.voice_cmd_route_not_found).format(name)) } } } @@ -686,16 +671,11 @@ class SoundscapeService : MediaSessionService() { coroutineScope.launch { val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService val markers = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllMarkers() - if (requestAudioFocus()) { - if (markers.isEmpty()) { - audioEngine.createTextToSpeech(ctx.getString(R.string.voice_cmd_no_markers), AudioType.STANDARD) - } else { - val names = markers.joinToString(", ") { it.name } - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_markers_list) + names, - AudioType.STANDARD - ) - } + if (markers.isEmpty()) { + speak2dText(ctx.getString(R.string.voice_cmd_no_markers)) + } else { + val names = markers.joinToString(", ") { it.name } + speak2dText(ctx.getString(R.string.voice_cmd_markers_list) + names) } } } @@ -720,21 +700,12 @@ class SoundscapeService : MediaSessionService() { } if (bestId != -1L) { val marker = markers.first { it.markerId == bestId } - if (requestAudioFocus()) { - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name), - AudioType.STANDARD - ) - } + speak2dText(ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name)) + val location = LngLatAlt(marker.longitude, marker.latitude) startBeacon(location, marker.name) } else { - if (requestAudioFocus()) { - audioEngine.createTextToSpeech( - ctx.getString(R.string.voice_cmd_marker_not_found).format(name), - AudioType.STANDARD - ) - } + speak2dText(ctx.getString(R.string.voice_cmd_marker_not_found).format(name)) } } } @@ -756,24 +727,36 @@ class SoundscapeService : MediaSessionService() { heading: Double = Double.NaN) { if (!requestAudioFocus()) { - Log.w(TAG, "speakText: Could not get audio focus. Aborting callouts.") + Log.w(TAG, "speakText: Could not get audio focus.") return } Log.d(TAG, "speakText $text") audioEngine.createTextToSpeech(text, type, latitude, longitude, heading) } + fun speak2dText(text: String, clearQueue: Boolean = false, earcon: String? = null) { + if (!requestAudioFocus()) { + Log.w(TAG, "speak2dText: Could not get audio focus.") + return + } + if(clearQueue) + audioEngine.clearTextToSpeechQueue() + if(earcon != null) { + audioEngine.createEarcon(earcon, AudioType.STANDARD) + } + audioEngine.createTextToSpeech(text, AudioType.STANDARD) + } fun speakCallout(callout: TrackedCallout?, addModeEarcon: Boolean) { if(callout == null) return if (!requestAudioFocus()) { - Log.w(TAG, "SpeakCallout: Could not get audio focus. Aborting callouts.") + Log.w(TAG, "SpeakCallout: Could not get audio focus.") return } - if(addModeEarcon) audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_ENTER, AudioType.STANDARD) + if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_ENTER, AudioType.STANDARD) for(result in callout.positionedStrings) { if(result.location == null) { var type = result.type @@ -801,7 +784,7 @@ class SoundscapeService : MediaSessionService() { ) } } - if(addModeEarcon) audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_EXIT, AudioType.STANDARD) + if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_EXIT, AudioType.STANDARD) callout.calloutHistory?.add(callout) callout.locationFilter?.update(callout.userGeometry) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt index 2aac7fde..0a339fc5 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt @@ -10,7 +10,8 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.AudioType -import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_ENTER +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_EXIT import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt import org.scottishtecharmy.soundscape.services.SoundscapeService @@ -57,7 +58,6 @@ class AudioMenu( } // ── Navigation state ────────────────────────────────────────────────────── - private data class MenuLevel(val items: List, var currentIndex: Int) /** Stack of levels; the root is at index 0, deeper sub-menus higher up. */ @@ -83,7 +83,7 @@ class AudioMenu( level.currentIndex = (level.currentIndex + 1) % level.items.size level.items[level.currentIndex].label } - speak(label) + service.speak2dText(label, true) } fun previous() { @@ -97,7 +97,7 @@ class AudioMenu( level.currentIndex - 1 level.items[level.currentIndex].label } - speak(label) + service.speak2dText(label, true) } fun select() { @@ -105,11 +105,6 @@ class AudioMenu( val item = synchronized(this) { menuStack.last().let { it.items[it.currentIndex] } } when (item) { is MenuItem.Action -> { - if (!service.requestAudioFocus()) { - Log.w(TAG, "select: could not get audio focus") - return - } - service.audioEngine.clearTextToSpeechQueue() item.action() } is MenuItem.Submenu -> { @@ -117,7 +112,7 @@ class AudioMenu( menuStack.addLast(MenuLevel(item.children, 0)) item.children[0].label } - speakWithEnterEarcon(firstLabel) + service.speak2dText(firstLabel, true, EARCON_MODE_ENTER) } is MenuItem.DynamicSubmenu -> loadAndEnter(item) } @@ -145,34 +140,10 @@ class AudioMenu( Log.w(TAG, "resetToRoot: could not get audio focus") return } - service.audioEngine.clearTextToSpeechQueue() - service.audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_EXIT, AudioType.STANDARD) - service.audioEngine.createTextToSpeech(firstRootLabel, AudioType.STANDARD) + service.speak2dText(firstRootLabel, true, EARCON_MODE_EXIT) } // ── Audio helpers ───────────────────────────────────────────────────────── - - /** Announce a label with no earcon — used for plain navigation. */ - private fun speak(label: String) { - if (!service.requestAudioFocus()) { - Log.w(TAG, "speak: could not get audio focus") - return - } - service.audioEngine.clearTextToSpeechQueue() - service.audioEngine.createTextToSpeech(label, AudioType.STANDARD) - } - - /** Announce a label preceded by the mode-enter earcon — used when descending into a sub-menu. */ - private fun speakWithEnterEarcon(label: String) { - if (!service.requestAudioFocus()) { - Log.w(TAG, "speakWithEnterEarcon: could not get audio focus") - return - } - service.audioEngine.clearTextToSpeechQueue() - service.audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_ENTER, AudioType.STANDARD) - service.audioEngine.createTextToSpeech(label, AudioType.STANDARD) - } - private fun loadAndEnter(item: MenuItem.DynamicSubmenu) { scope.launch { val children = item.childrenProvider() @@ -183,7 +154,7 @@ class AudioMenu( menuStack.addLast(MenuLevel(children, 0)) children[0].label } - speakWithEnterEarcon(firstLabel) + service.speak2dText(firstLabel, true, EARCON_MODE_ENTER) } } } @@ -192,7 +163,7 @@ class AudioMenu( val label = localizedContext.getString(id) return MenuItem.Action(label) { applyAudioProfile(profile) - service.speakText(label, AudioType.STANDARD) + service.speak2dText(label) } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index 2ad2b0fb..3da5fa79 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -10,8 +10,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import org.scottishtecharmy.soundscape.R -import org.scottishtecharmy.soundscape.audio.AudioType -import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_CALLOUTS_OFF import org.scottishtecharmy.soundscape.services.SoundscapeService import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale @@ -150,16 +149,11 @@ class VoiceCommandManager( println("Found command: ${context.getString(bestMatch.stringId)}") bestMatch.action(t) } else { - if (service.requestAudioFocus()) { - service.audioEngine.createEarcon( - NativeAudioEngine.EARCON_CALLOUTS_OFF, - AudioType.STANDARD - ) - service.audioEngine.createTextToSpeech( - context.getString(R.string.voice_cmd_not_recognized).format(t), - AudioType.STANDARD - ) - } + service.speak2dText( + context.getString(R.string.voice_cmd_not_recognized).format(t), + false, + EARCON_CALLOUTS_OFF + ) } } @@ -167,8 +161,6 @@ class VoiceCommandManager( val commandNames = commands.map { context.getString(it.stringId) } val text = context.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(", ") - if (service.requestAudioFocus()) { - service.audioEngine.createTextToSpeech(text, AudioType.STANDARD) - } + service.speak2dText(text) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index d74002a2..8bdac501 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -479,17 +479,9 @@ "Hear about places in front of you" -"Select audio profile" -"Eating" -"Shopping" -"Navigating" -"Roads only" -"No filtering" "Route control" "Next waypoint" "Previous waypoint" -"Stop" -"Choose route to start" "No routes saved" "Top menu" From 940163ea2d6349bcb997deb01b491b994734ea7b Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Sat, 28 Feb 2026 15:26:45 +0000 Subject: [PATCH 12/18] Add biasing strings to speech recognition All possible voice commands along with marker and route names are passed into the SpeechRecognizer so that it knows what to expect. --- .../soundscape/services/SoundscapeService.kt | 29 ++++++++++----- .../mediacontrol/VoiceCommandManager.kt | 35 +++++++++++++------ app/src/main/res/values/strings.xml | 2 +- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 065dd940..34ae9651 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.flow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -312,6 +313,16 @@ class SoundscapeService : MediaSessionService() { onError = { } ) + // Keep biasing strings up to date whenever markers or routes change + val dao = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao() + coroutineScope.launch { + combine(dao.getAllMarkersFlow(), dao.getAllRoutesFlow()) { markers, routes -> + markers.map { it.name } + routes.map { it.name } + }.collect { names -> + voiceCommandManager.updateBiasingStrings(names) + } + } + mediaSession = MediaSession.Builder(this, mediaPlayer) .setId("org.scottishtecharmy.soundscape") .setCallback(SoundscapeMediaSessionCallback { mediaControlsTarget }) @@ -574,14 +585,15 @@ class SoundscapeService : MediaSessionService() { val ctx = if (::localizedContext.isInitialized) localizedContext else this - // Inform the user that we're listening - speak2dText(ctx.getString(R.string.voice_cmd_listening), true, EARCON_CALLOUTS_ON) + // Play earcon as feedback to indicate that we're starting to listen. + speak2dText("", true, EARCON_CALLOUTS_ON) - // Wait for the TTS to finish before opening the mic + // Wait for the TTS to finish before opening the mic, otherwise we lose audio focus before + // the icon completes. coroutineScope.launch { - val deadline = System.currentTimeMillis() + 3_000L + val deadline = System.currentTimeMillis() + 1000L while (isAudioEngineBusy() && System.currentTimeMillis() < deadline) { - delay(50) + delay(20) } withContext(Dispatchers.Main) { voiceCommandManager.startListening() @@ -633,7 +645,7 @@ class SoundscapeService : MediaSessionService() { if (routes.isEmpty()) speak2dText(ctx.getString(R.string.voice_cmd_no_routes)) else { - val names = routes.joinToString(", ") { it.name } + val names = routes.joinToString(". ") { it.name } speak2dText(ctx.getString(R.string.voice_cmd_routes_list) + names) } } @@ -674,7 +686,7 @@ class SoundscapeService : MediaSessionService() { if (markers.isEmpty()) { speak2dText(ctx.getString(R.string.voice_cmd_no_markers)) } else { - val names = markers.joinToString(", ") { it.name } + val names = markers.joinToString(". ") { it.name } speak2dText(ctx.getString(R.string.voice_cmd_markers_list) + names) } } @@ -744,7 +756,8 @@ class SoundscapeService : MediaSessionService() { if(earcon != null) { audioEngine.createEarcon(earcon, AudioType.STANDARD) } - audioEngine.createTextToSpeech(text, AudioType.STANDARD) + if(text.isNotEmpty()) + audioEngine.createTextToSpeech(text, AudioType.STANDARD) } fun speakCallout(callout: TrackedCallout?, addModeEarcon: Boolean) { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index 3da5fa79..c8927706 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -2,6 +2,7 @@ package org.scottishtecharmy.soundscape.services.mediacontrol import android.content.Context import android.content.Intent +import android.os.Build import android.os.Bundle import android.speech.RecognitionListener import android.speech.RecognizerIntent @@ -30,12 +31,18 @@ class VoiceCommandManager( private var speechRecognizer: SpeechRecognizer? = null private val _state = MutableStateFlow(VoiceCommandState.Idle) val state: StateFlow = _state.asStateFlow() + @Volatile private var extraBiasingStrings: List = emptyList() /** Call this whenever SoundscapeService updates its localizedContext. */ fun updateContext(newContext: Context) { context = newContext } + /** Call this with current marker/route names whenever they are updated. */ + fun updateBiasingStrings(strings: List) { + extraBiasingStrings = strings + } + // Must be called on the main thread (satisfied: service is on main thread) fun startListening() { if (_state.value is VoiceCommandState.Listening) return @@ -60,6 +67,12 @@ class VoiceCommandManager( // https://medium.com/@andraz.pajtler/android-speech-to-text-the-missing-guide-part-1-824e2636c45a // Match recognizer language to the app's configured locale putExtra(RecognizerIntent.EXTRA_LANGUAGE, getCurrentLocale().toLanguageTag()) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val biasingStrings = ArrayList() + commands.forEach { biasingStrings.add(context.getString(it.stringId)) } + biasingStrings.addAll(extraBiasingStrings) + putStringArrayListExtra(RecognizerIntent.EXTRA_BIASING_STRINGS, biasingStrings) + } } speechRecognizer?.startListening(intent) } @@ -105,26 +118,26 @@ class VoiceCommandManager( val commands = arrayOf( VoiceCommand(R.string.directions_my_location) { service.myLocation() }, - VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, - VoiceCommand(R.string.help_explore_page_title) { service.aheadOfMe() }, - VoiceCommand(R.string.callouts_nearby_markers) { service.nearbyMarkers() }, + VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, + VoiceCommand(R.string.help_explore_page_title) { service.aheadOfMe() }, + VoiceCommand(R.string.callouts_nearby_markers) { service.nearbyMarkers() }, VoiceCommand(R.string.route_detail_action_next) { service.routeSkipNext() }, VoiceCommand(R.string.route_detail_action_previous) { service.routeSkipPrevious() }, - VoiceCommand(R.string.beacon_action_mute_beacon) { service.routeMute() }, - VoiceCommand(R.string.route_detail_action_stop_route) { service.routeStop() }, - VoiceCommand(R.string.voice_cmd_list_routes) { service.routeListRoutes() }, - VoiceCommand(R.string.route_detail_action_start_route) { + VoiceCommand(R.string.beacon_action_mute_beacon) { service.routeMute() }, + VoiceCommand(R.string.route_detail_action_stop_route) { service.routeStop() }, + VoiceCommand(R.string.voice_cmd_list_routes) { service.routeListRoutes() }, + VoiceCommand(R.string.route_detail_action_start_route) { // TODO: We need a "fuzzy remove" here val routeName = it.removePrefix(context.getString(R.string.route_detail_action_start_route).lowercase()).trim() service.routeStartByName(routeName) }, - VoiceCommand(R.string.voice_cmd_list_markers) { service.routeListMarkers() }, + VoiceCommand(R.string.voice_cmd_list_markers) { service.routeListMarkers() }, VoiceCommand(R.string.voice_cmd_start_beacon_at_marker) { // TODO: We need a "fuzzy remove" here - val markerName = it.removePrefix(context.getString(R.string.location_detail_action_beacon).lowercase()).trim() + val markerName = it.removePrefix(context.getString(R.string.voice_cmd_start_beacon_at_marker).lowercase()).trim() service.markerStartByName(markerName) }, - VoiceCommand(R.string.menu_help) { voiceHelp() }, + VoiceCommand(R.string.menu_help) { voiceHelp() }, ) private fun handleSpeech(speech: ArrayList) { @@ -160,7 +173,7 @@ class VoiceCommandManager( private fun voiceHelp() { val commandNames = commands.map { context.getString(it.stringId) } val text = - context.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(", ") + context.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(". ") service.speak2dText(text) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8bdac501..cd5ee5d5 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2618,7 +2618,7 @@ Please report any problems that you have, however small they may be via the *Con Starting route %s -Start audio beacon at %s +Start audio beacon at Starting audio beacon at %s From 7d2307a494cdf007a185b47c7abfd3611d7ceac4 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Mon, 2 Mar 2026 11:10:19 +0000 Subject: [PATCH 13/18] Add onboarding of record audio permissions for voice control commands Fallback to audio menu if no permission granted. --- .../onboarding/navigating/NavigatingScreen.kt | 37 +++++++++++++++++-- .../soundscape/services/SoundscapeService.kt | 37 +++++++++++-------- app/src/main/res/values/strings.xml | 4 ++ 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt index b51c1f61..d1cdbe44 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt @@ -19,6 +19,7 @@ import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.LocationOn +import androidx.compose.material.icons.rounded.Mic import androidx.compose.material.icons.rounded.Notifications import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme @@ -51,12 +52,14 @@ fun NavigatingScreen( arrayOf( Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_BACKGROUND_LOCATION, - Manifest.permission.POST_NOTIFICATIONS + Manifest.permission.POST_NOTIFICATIONS, + Manifest.permission.RECORD_AUDIO ) } else { arrayOf( Manifest.permission.ACCESS_FINE_LOCATION, - Manifest.permission.ACCESS_BACKGROUND_LOCATION + Manifest.permission.ACCESS_BACKGROUND_LOCATION, + Manifest.permission.RECORD_AUDIO ) } val multiplePermissionResultLauncher = rememberLauncherForActivityResult( @@ -135,7 +138,7 @@ fun Navigating( contentDescription = null, tint = MaterialTheme.colorScheme.onSurface ) - Spacer(modifier = Modifier.width(spacing.medium)) + Spacer(modifier = Modifier.width(spacing.extraSmall)) Column( modifier = Modifier.semantics(mergeDescendants = true) {}, ) { @@ -182,7 +185,33 @@ fun Navigating( ) } } - + } + Row( + modifier = Modifier + .mediumPadding() + .fillMaxWidth(), + ) + { + Icon( + Icons.Rounded.Mic, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurface + ) + Spacer(modifier = Modifier.width(spacing.extraSmall)) + Column( + modifier = Modifier.semantics(mergeDescendants = true) {}, + ) { + Text( + text = stringResource(R.string.first_launch_permissions_record_audio), + style = MaterialTheme.typography.titleMedium, + color = MaterialTheme.colorScheme.onSurface, + ) + Text( + text = stringResource(R.string.first_launch_permissions_required_for_voice_control), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface, + ) + } } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 34ae9651..ba48ebf7 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -174,10 +174,10 @@ class SoundscapeService : MediaSessionService() { private val _gridStateFlow = MutableStateFlow(null) var gridStateFlow: StateFlow = _gridStateFlow - // Voice command manager - private lateinit var voiceCommandManager: VoiceCommandManager + // Voice command manager — only initialized when RECORD_AUDIO permission is granted + private var voiceCommandManager: VoiceCommandManager? = null val voiceCommandStateFlow: StateFlow - get() = voiceCommandManager.state + get() = voiceCommandManager?.state ?: MutableStateFlow(VoiceCommandState.Idle) // Media control button code private var mediaSession: MediaSession? = null @@ -263,7 +263,7 @@ class SoundscapeService : MediaSessionService() { val configuration = Configuration(applicationContext.resources.configuration) configuration.setLocale(configLocale) localizedContext = applicationContext.createConfigurationContext(configuration) - if (::voiceCommandManager.isInitialized) voiceCommandManager.updateContext(localizedContext) + voiceCommandManager?.updateContext(localizedContext) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) started = true } @@ -303,23 +303,26 @@ class SoundscapeService : MediaSessionService() { // create new RealmDB or open existing startRealms(applicationContext) + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + == PackageManager.PERMISSION_GRANTED) { + voiceCommandManager = VoiceCommandManager( + service = this, + onError = { } + ) + } + // Update the media controls mode val sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) val mode = sharedPreferences.getString(MEDIA_CONTROLS_MODE_KEY, MEDIA_CONTROLS_MODE_DEFAULT)!! updateMediaControls(mode) - voiceCommandManager = VoiceCommandManager( - service = this, - onError = { } - ) - // Keep biasing strings up to date whenever markers or routes change val dao = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao() coroutineScope.launch { combine(dao.getAllMarkersFlow(), dao.getAllRoutesFlow()) { markers, routes -> markers.map { it.name } + routes.map { it.name } }.collect { names -> - voiceCommandManager.updateBiasingStrings(names) + voiceCommandManager?.updateBiasingStrings(names) } } @@ -331,13 +334,15 @@ class SoundscapeService : MediaSessionService() { } fun updateMediaControls(target: String) { - mediaControlsTarget = when(target) { - "Original" -> OriginalMediaControls(this) - "VoiceControl" -> VoiceCommandMediaControls(this) + val hasRecordAudio = ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + mediaControlsTarget = when (target) { + "VoiceControl" if hasRecordAudio -> VoiceCommandMediaControls(this) + "VoiceControl" -> AudioMenuMediaControls(audioMenu) "AudioMenu" -> AudioMenuMediaControls(audioMenu) + "Original" -> OriginalMediaControls(this) else -> OriginalMediaControls(this) } - } override fun onTaskRemoved(rootIntent: Intent?) { @@ -377,7 +382,7 @@ class SoundscapeService : MediaSessionService() { wakeLock?.let { if (it.isHeld) it.release() } wakeLock = null - if (::voiceCommandManager.isInitialized) voiceCommandManager.destroy() + voiceCommandManager?.destroy() // Clear service reference in binder so that it can be garbage collected binder?.reset() @@ -596,7 +601,7 @@ class SoundscapeService : MediaSessionService() { delay(20) } withContext(Dispatchers.Main) { - voiceCommandManager.startListening() + voiceCommandManager?.startListening() } } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index cd5ee5d5..69c7d2ff 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -836,6 +836,10 @@ No language selected Notifications + +Record audio + +This permission is optional, but is required if you want to use voice commands to control the app. The Soundscape location service runs even when the phone is locked. From fa456fd2b6e51f074e86c8a24ddeb759bb3dfb34 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Mon, 2 Mar 2026 12:38:30 +0000 Subject: [PATCH 14/18] Improve route and beacon starting from voice control This new approach should work better across different languages. We compare complete commands including marker and route names rather than trying to split them up into command + name. This should handle languages where the ordering of the command and names isn't simple. --- .../soundscape/SoundscapeServiceConnection.kt | 2 +- .../soundscape/services/SoundscapeService.kt | 78 +++------- .../services/mediacontrol/AudioMenu.kt | 2 +- .../mediacontrol/VoiceCommandManager.kt | 141 +++++++++++++----- app/src/main/res/values/strings.xml | 10 +- 5 files changed, 134 insertions(+), 99 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt index ae36f3cd..8d404242 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt @@ -59,7 +59,7 @@ class SoundscapeServiceConnection @Inject constructor() { } fun routeStart(routeId: Long) { - soundscapeService?.routeStart(routeId) + soundscapeService?.routeStartById(routeId) } fun startBeacon(location: LngLatAlt, name: String) { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index ba48ebf7..91f2bd67 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -53,6 +53,8 @@ import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_ import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_ENTER import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_EXIT import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase +import org.scottishtecharmy.soundscape.database.local.model.MarkerEntity +import org.scottishtecharmy.soundscape.database.local.model.RouteEntity import org.scottishtecharmy.soundscape.geoengine.GeoEngine import org.scottishtecharmy.soundscape.geoengine.GridState import org.scottishtecharmy.soundscape.geoengine.StreetPreviewChoice @@ -80,7 +82,6 @@ import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandManager import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandMediaControls import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import org.scottishtecharmy.soundscape.utils.Analytics -import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds @@ -319,10 +320,13 @@ class SoundscapeService : MediaSessionService() { // Keep biasing strings up to date whenever markers or routes change val dao = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao() coroutineScope.launch { - combine(dao.getAllMarkersFlow(), dao.getAllRoutesFlow()) { markers, routes -> - markers.map { it.name } + routes.map { it.name } - }.collect { names -> - voiceCommandManager?.updateBiasingStrings(names) + dao.getAllMarkersFlow().collect { markers -> + voiceCommandManager?.updateMarkers(markers) + } + } + coroutineScope.launch { + dao.getAllRoutesFlow().collect { routes -> + voiceCommandManager?.updateRoutes(routes) } } @@ -617,7 +621,7 @@ class SoundscapeService : MediaSessionService() { fun startBeacon(location: LngLatAlt, name: String) { routePlayer.startBeacon(location, name) } - fun routeStart(routeId: Long) { + fun routeStartById(routeId: Long) { routePlayer.startRoute(routeId) } fun routeStop() { @@ -656,32 +660,10 @@ class SoundscapeService : MediaSessionService() { } } - fun routeStartByName(name: String) { - if (name.isEmpty()) { - routeListRoutes() - return - } - coroutineScope.launch { - val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService - val routes = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllRoutes() - val nameLower = name.lowercase() - var bestId = -1L - var bestScore = Double.MAX_VALUE - for (route in routes) { - val score = nameLower.fuzzyCompare(route.name.lowercase(), true) - if (score < 0.4 && score < bestScore) { - bestScore = score - bestId = route.routeId - } - } - if (bestId != -1L) { - val routeName = routes.first { it.routeId == bestId }.name - speak2dText(ctx.getString(R.string.voice_cmd_starting_route).format(routeName)) - routeStart(bestId) - } else { - speak2dText(ctx.getString(R.string.voice_cmd_route_not_found).format(name)) - } - } + fun routeStart(route: RouteEntity) { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + speak2dText(ctx.getString(R.string.voice_cmd_starting_route).format(route.name)) + routeStartById(route.routeId) } fun routeListMarkers() { @@ -697,34 +679,12 @@ class SoundscapeService : MediaSessionService() { } } - fun markerStartByName(name: String) { - if (name.isEmpty()) { - routeListMarkers() - return - } - coroutineScope.launch { - val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService - val markers = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllMarkers() - val nameLower = name.lowercase() - var bestId = -1L - var bestScore = Double.MAX_VALUE - for (marker in markers) { - val score = nameLower.fuzzyCompare(marker.name.lowercase(), true) - if (score < 0.4 && score < bestScore) { - bestScore = score - bestId = marker.markerId - } - } - if (bestId != -1L) { - val marker = markers.first { it.markerId == bestId } - speak2dText(ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name)) + fun markerStart(marker: MarkerEntity) { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + speak2dText(ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name)) - val location = LngLatAlt(marker.longitude, marker.latitude) - startBeacon(location, marker.name) - } else { - speak2dText(ctx.getString(R.string.voice_cmd_marker_not_found).format(name)) - } - } + val location = LngLatAlt(marker.longitude, marker.latitude) + startBeacon(location, marker.name) } /** diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt index 0a339fc5..6db49c0e 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt @@ -226,7 +226,7 @@ class AudioMenu( withContext(Dispatchers.IO) { val db = MarkersAndRoutesDatabase.getMarkersInstance(service) db.routeDao().getAllRoutes().map { route -> - MenuItem.Action(route.name) { service.routeStart(route.routeId) } + MenuItem.Action(route.name) { service.routeStartById(route.routeId) } } + mainMenuAction() } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index c8927706..7d20d4af 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -12,6 +12,8 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_CALLOUTS_OFF +import org.scottishtecharmy.soundscape.database.local.model.MarkerEntity +import org.scottishtecharmy.soundscape.database.local.model.RouteEntity import org.scottishtecharmy.soundscape.services.SoundscapeService import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale @@ -31,16 +33,19 @@ class VoiceCommandManager( private var speechRecognizer: SpeechRecognizer? = null private val _state = MutableStateFlow(VoiceCommandState.Idle) val state: StateFlow = _state.asStateFlow() - @Volatile private var extraBiasingStrings: List = emptyList() + @Volatile private var listOfRoutes: List = emptyList() + @Volatile private var listOfMarkers: List = emptyList() /** Call this whenever SoundscapeService updates its localizedContext. */ fun updateContext(newContext: Context) { context = newContext } - /** Call this with current marker/route names whenever they are updated. */ - fun updateBiasingStrings(strings: List) { - extraBiasingStrings = strings + fun updateRoutes(routes: List) { + listOfRoutes = routes + } + fun updateMarkers(markers: List) { + listOfMarkers = markers } // Must be called on the main thread (satisfied: service is on main thread) @@ -69,8 +74,14 @@ class VoiceCommandManager( putExtra(RecognizerIntent.EXTRA_LANGUAGE, getCurrentLocale().toLanguageTag()) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val biasingStrings = ArrayList() - commands.forEach { biasingStrings.add(context.getString(it.stringId)) } - biasingStrings.addAll(extraBiasingStrings) + simpleCommands.forEach { biasingStrings.add(context.getString(it.stringId)) } + val markers = listOfMarkers + val routes = listOfRoutes + for(marker in markers) + biasingStrings.add(context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(marker.name)) + for(route in routes) + biasingStrings.add(context.getString(R.string.voice_cmd_start_route).format(route.name)) + putStringArrayListExtra(RecognizerIntent.EXTRA_BIASING_STRINGS, biasingStrings) } } @@ -114,9 +125,13 @@ class VoiceCommandManager( override fun onEvent(eventType: Int, params: Bundle?) {} } - data class VoiceCommand(val stringId: Int, val action: (arg: String) -> Unit) + data class VoiceCommand(val stringId: Int, val action: (arg: ArrayList) -> Unit) + + private fun getArgument(speech: ArrayList, commandString: String) : String { + return "" + } - val commands = arrayOf( + val simpleCommands = arrayOf( VoiceCommand(R.string.directions_my_location) { service.myLocation() }, VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, VoiceCommand(R.string.help_explore_page_title) { service.aheadOfMe() }, @@ -126,44 +141,85 @@ class VoiceCommandManager( VoiceCommand(R.string.beacon_action_mute_beacon) { service.routeMute() }, VoiceCommand(R.string.route_detail_action_stop_route) { service.routeStop() }, VoiceCommand(R.string.voice_cmd_list_routes) { service.routeListRoutes() }, - VoiceCommand(R.string.route_detail_action_start_route) { - // TODO: We need a "fuzzy remove" here - val routeName = it.removePrefix(context.getString(R.string.route_detail_action_start_route).lowercase()).trim() - service.routeStartByName(routeName) - }, VoiceCommand(R.string.voice_cmd_list_markers) { service.routeListMarkers() }, - VoiceCommand(R.string.voice_cmd_start_beacon_at_marker) { - // TODO: We need a "fuzzy remove" here - val markerName = it.removePrefix(context.getString(R.string.voice_cmd_start_beacon_at_marker).lowercase()).trim() - service.markerStartByName(markerName) - }, VoiceCommand(R.string.menu_help) { voiceHelp() }, ) - private fun handleSpeech(speech: ArrayList) { + fun matchDynamicMarkers(speech: String): Boolean{ + val markers = listOfMarkers + val routes = listOfRoutes - // TODO: Start by only looking at the very first string for a match. We should check the - // other strings too. + var minMatch = Double.MAX_VALUE - // Find the best match to the speech. - val t = speech.first().lowercase() + // Markers + var bestMarker : MarkerEntity? = null + for(marker in markers) { + val commandString = context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(marker.name).lowercase() + println("Marker compare \"$commandString\" with \"$speech\"") + val match = commandString.fuzzyCompare(speech, false) + if ((match < 0.2) && (match < minMatch)) { + minMatch = match + bestMarker = marker + } + } + if(bestMarker != null) { + service.markerStart(bestMarker) + return true + } - var minMatch = Double.MAX_VALUE - var bestMatch: VoiceCommand? = null - for (command in commands) { - val commandString = context.getString(command.stringId).lowercase() - val match = commandString.fuzzyCompare(t, true) - if ((match < 0.3) && (match < minMatch)) { + // Routes + var bestRoute : RouteEntity? = null + for(route in routes) { + val commandString = context.getString(R.string.voice_cmd_start_route).format(route.name).lowercase() + println("Route compare \"$commandString\" with \"$speech\"") + val match = commandString.fuzzyCompare(speech, false) + if ((match < 0.2) && (match < minMatch)) { minMatch = match - bestMatch = command + bestRoute = route + } + } + if(bestRoute != null) { + service.routeStart(bestRoute) + return true + } + + return false + } + + private fun handleSpeech(speech: ArrayList) { + var bestMatch: VoiceCommand? = null + for(text in speech) { + + // Find the best match to the speech. + val t = text.lowercase() + + var minMatch = Double.MAX_VALUE + // Start with simpleCommands which don't contain any dynamic arguments + for (command in simpleCommands) { + val commandString = context.getString(command.stringId).lowercase() + val match = commandString.fuzzyCompare(t, false) + if ((match < 0.3) && (match < minMatch)) { + minMatch = match + bestMatch = command + } } + if(bestMatch != null) + break + println("No simple command match found") + + // Check if we match any of our dynamic markers + if(matchDynamicMarkers(t)) + return } if (bestMatch != null) { println("Found command: ${context.getString(bestMatch.stringId)}") - bestMatch.action(t) + + // Pass in all the speech strings, it may be that the argument is clearer in ones other + // than our best match. + bestMatch.action(speech) } else { service.speak2dText( - context.getString(R.string.voice_cmd_not_recognized).format(t), + context.getString(R.string.voice_cmd_not_recognized).format(speech.first()), false, EARCON_CALLOUTS_OFF ) @@ -171,9 +227,22 @@ class VoiceCommandManager( } private fun voiceHelp() { - val commandNames = commands.map { context.getString(it.stringId) } - val text = - context.getString(R.string.voice_cmd_help_response) + commandNames.joinToString(". ") - service.speak2dText(text) + + val firstRoute = listOfRoutes.firstOrNull()?.name + val firstMarker = listOfMarkers.firstOrNull()?.name + + val commandNames = simpleCommands.map { context.getString(it.stringId) } + val builder = StringBuilder() + builder.append(context.getString(R.string.voice_cmd_help_response)) + commandNames.forEach { builder.append(it).append(". ") } + + if((firstRoute != null) || (firstMarker != null)) + builder.append(context.getString(R.string.voice_cmd_explain_dynamic_markers)) + if(firstRoute != null) + builder.append(context.getString(R.string.voice_cmd_start_route).format(firstRoute)).append(". ") + if(firstMarker != null) + builder.append(context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(firstMarker)).append(". ") + + service.speak2dText(builder.toString()) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 69c7d2ff..cfb9aa79 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2621,10 +2621,16 @@ Please report any problems that you have, however small they may be via the *Con Route not found: %s Starting route %s - -Start audio beacon at + +Start route %s + + + +Start audio beacon at %s Starting audio beacon at %s Marker not found: %s + +You can also start routes and markers by name, for example. From 85853c54177f43af99d8f7b36bb6f8f632857629 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Mon, 2 Mar 2026 13:23:34 +0000 Subject: [PATCH 15/18] Only set language for SpeechRecognizer if it's one that is supported Some extra work goes on here so that we try and pick the best language and region. --- .../soundscape/services/SoundscapeService.kt | 5 +- .../mediacontrol/VoiceCommandManager.kt | 85 ++++++++++++++----- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 91f2bd67..a5b04cc1 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -341,7 +341,10 @@ class SoundscapeService : MediaSessionService() { val hasRecordAudio = ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED mediaControlsTarget = when (target) { - "VoiceControl" if hasRecordAudio -> VoiceCommandMediaControls(this) + "VoiceControl" if hasRecordAudio -> { + voiceCommandManager?.initialize() + VoiceCommandMediaControls(this) + } "VoiceControl" -> AudioMenuMediaControls(audioMenu) "AudioMenu" -> AudioMenuMediaControls(audioMenu) "Original" -> OriginalMediaControls(this) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt index 7d20d4af..43bb12e1 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -5,8 +5,11 @@ import android.content.Intent import android.os.Build import android.os.Bundle import android.speech.RecognitionListener +import android.speech.RecognitionSupport +import android.speech.RecognitionSupportCallback import android.speech.RecognizerIntent import android.speech.SpeechRecognizer +import androidx.core.content.ContextCompat import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -17,6 +20,7 @@ import org.scottishtecharmy.soundscape.database.local.model.RouteEntity import org.scottishtecharmy.soundscape.services.SoundscapeService import org.scottishtecharmy.soundscape.utils.fuzzyCompare import org.scottishtecharmy.soundscape.utils.getCurrentLocale +import java.util.Locale sealed class VoiceCommandState { object Idle : VoiceCommandState() @@ -35,6 +39,8 @@ class VoiceCommandManager( val state: StateFlow = _state.asStateFlow() @Volatile private var listOfRoutes: List = emptyList() @Volatile private var listOfMarkers: List = emptyList() + // Language tag validated against the recognizer's supported list; set by initialize(). + private var cachedLanguage: String? = null /** Call this whenever SoundscapeService updates its localizedContext. */ fun updateContext(newContext: Context) { @@ -48,30 +54,71 @@ class VoiceCommandManager( listOfMarkers = markers } - // Must be called on the main thread (satisfied: service is on main thread) - fun startListening() { - if (_state.value is VoiceCommandState.Listening) return + // Must be called on the main thread when switching to VoiceControl mode. + // Creates the SpeechRecognizer and pre-fetches the supported language list so that + // startListening() can start immediately without an extra round-trip. + fun initialize() { + destroyRecognizer() + cachedLanguage = null + if (!SpeechRecognizer.isRecognitionAvailable(context)) { println("Recognition is unavailable") onError() return } - if (speechRecognizer == null) { - speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) - speechRecognizer?.setRecognitionListener(listener) - } - val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra( - RecognizerIntent.EXTRA_LANGUAGE_MODEL, - RecognizerIntent.LANGUAGE_MODEL_FREE_FORM + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) + speechRecognizer?.setRecognitionListener(listener) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val probeIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + } + speechRecognizer?.checkRecognitionSupport( + probeIntent, + ContextCompat.getMainExecutor(context), + object : RecognitionSupportCallback { + override fun onSupportResult(recognitionSupport: RecognitionSupport) { + val supported = recognitionSupport.installedOnDeviceLanguages + + recognitionSupport.onlineLanguages + supported.forEach { println("Supported language: $it") } + cachedLanguage = pickBestLanguage(getCurrentLocale(), supported) + println("cachedLanguage $cachedLanguage") + } + + override fun onError(error: Int) { + // Support query failed — cachedLanguage stays null (no EXTRA_LANGUAGE). + } + } ) + } + } + + // Must be called on the main thread (satisfied: service is on main thread) + fun startListening() { + if (_state.value is VoiceCommandState.Listening) return + speechRecognizer?.startListening(buildRecognitionIntent(cachedLanguage)) + } + + /** + * Choose the best BCP-47 language tag from [supportedTags] for the given [locale]. + * Prefers an exact match (language + country); falls back to any tag with the same language + * code; returns null if nothing matches. + */ + private fun pickBestLanguage(locale: Locale, supportedTags: List): String? { + + val tag = locale.toLanguageTag() + println("pickBestLanguage for $tag") + if (tag in supportedTags) return tag + val langCode = locale.language + return supportedTags.firstOrNull { Locale.forLanguageTag(it).language == langCode } + } + + private fun buildRecognitionIntent(language: String?): Intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) - // TODO: We need to query the API to find out which Locales are supported and use one of - // those. For example, we might want to use es_ES even if we're in another country. - // https://medium.com/@andraz.pajtler/android-speech-to-text-the-missing-guide-part-1-824e2636c45a - // Match recognizer language to the app's configured locale - putExtra(RecognizerIntent.EXTRA_LANGUAGE, getCurrentLocale().toLanguageTag()) + if (language != null) putExtra(RecognizerIntent.EXTRA_LANGUAGE, language) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val biasingStrings = ArrayList() simpleCommands.forEach { biasingStrings.add(context.getString(it.stringId)) } @@ -85,8 +132,6 @@ class VoiceCommandManager( putStringArrayListExtra(RecognizerIntent.EXTRA_BIASING_STRINGS, biasingStrings) } } - speechRecognizer?.startListening(intent) - } fun destroy() { destroyRecognizer() @@ -127,10 +172,6 @@ class VoiceCommandManager( data class VoiceCommand(val stringId: Int, val action: (arg: ArrayList) -> Unit) - private fun getArgument(speech: ArrayList, commandString: String) : String { - return "" - } - val simpleCommands = arrayOf( VoiceCommand(R.string.directions_my_location) { service.myLocation() }, VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, From c1920efb567713d984c967016cff9e03de08d824 Mon Sep 17 00:00:00 2001 From: davecraig Date: Fri, 27 Feb 2026 17:31:19 +0000 Subject: [PATCH 16/18] Add document describing voice control and audio menu control Also fix dokka build which was broken due to an incompatibility between AGP and the plugin. --- app/build.gradle.kts | 8 ++ docs/developers/voice-and-audio-control.md | 124 +++++++++++++++++++++ gradle/libs.versions.toml | 2 +- 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 docs/developers/voice-and-audio-control.md diff --git a/app/build.gradle.kts b/app/build.gradle.kts index abb057d3..212c8432 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -367,3 +367,11 @@ dependencies { testImplementation(libs.json) } + +dokka { + dokkaSourceSets.configureEach { + if (name == "main") { + suppress.set(true) + } + } +} diff --git a/docs/developers/voice-and-audio-control.md b/docs/developers/voice-and-audio-control.md new file mode 100644 index 00000000..80e5f460 --- /dev/null +++ b/docs/developers/voice-and-audio-control.md @@ -0,0 +1,124 @@ +--- +title: App remote control +layout: page +parent: Information for developers +has_toc: false +--- +# App remote control +As well as the accessible GUI to control the app, we also have 3 other options which can control the app even when the screen is locked: + +1. Hard coded media controls as per the iOS app +2. Voice control +3. An audio menu navigated via media controls + +The first of these was implemented in an early release, but the second two are new. + +## Voice control +Most voice commands will be relatively fixed and discoverable simply by asking. The expected design is that the user will tap the play/pause button on their headphones and that will prompt the user that it's listening for a command. The user speaks the command and after a short period of silence the listening will stop and the text is sent to the app to process. + +It's possible to have a number of phrases that have the same action, though it's not clear whether that really makes sense. If you can use "My location" to trigger that feature, then having "Where am I" as well might just be confusing? In general sticking to the same text as is used in the GUI makes the most sense. To do that, we should actually use the same string resources so that the translations remain consistent. + +Because we want the functionality of voice control and audio menus to be similar, I've added in the text alongside the audio menu options in the next section. + +## Audio menus +Each menu consists of a number of options to pick from. The main menu is simply a list of these sub menus. Each sub menu has an option which allows navigating back up to the main menu. That option obviously doesn't exist as a voice control command where all commands are in a flat structure. + +### Existing features +#### My Surroundings +Selecting an option simply triggers in the same way as tapping the UI button. +``` +My Surroundings +├── "My Location" +├── "Around Me" +├── "Ahead of Me" +├── "Nearby Markers" +└── Main Menu +``` + +#### Route control +These are the actions that are hard coded in iOS when a route is playing back. All button presses must have obvious audio feedback even if it's just to say "You're already at the first waypoint". +``` +Route +├── "Next Waypoint" +├── "Prev Waypoint" +├── "Mute Beacon" +├── "Stop Route" +└── Main Menu +``` + +#### Start route +This is the first of our dynamically generated menus. It contains every route that has been set up. The routes should be sorted alphabetically. +``` +Start Route +├── [route 1] +├── [route 2] +├── ... +├── ... +└── Main Menu +``` + +Two voice commands cover this: +1. "List routes" +2. "Start route [route name]" + +This means that the user can discover the available routes and then start playing one back. + +#### Start beacon +This is the same as start route, but for Markers. Many users don't create routes and so have fewer markers to scroll through. The markers should be sorted alphabetically. +``` +Start Beacon +├── [marker 1] +├── [marker 2] +├── ... +├── ... +└── Main Menu +``` + +Again, two voice commands cover this: +1. "List markers" +2. "Start beacon at [marker name]" + +Perhaps we could add filters to "List markers" e.g. "List markers nearby" or "List markers from F" where it start alphabetically. + +### New features +One of the main reasons for having an extensible menu system is so that we can add in new remotely controlled features. We need to decide how we expose these within the on screen GUI too - in many ways it's a lot easier not to! + +#### Audio profile +This is a new feature which would allow the user to pick a filter for points of interest. I think the implementation of the filters is the crucial thing here. Rather than ONLY playing out eating POI, perhaps it guarantees eating POI, but will play some other POI to aid navigation e.g. at least one POI per 100m if there have been no intersections? +``` +Audio Profile +├── "Eating" +├── "Shopping" +├── "Banking" +├── "Transport" +├── "Navigating" +├── ... +├── ... +└── Main Menu +``` + +#### Route creation +The UI for creating routes is hard to use, and the idea here is to allow users to create routes as they walk. +``` +Route creation +├── "Start creating route" +├── "Add waypoint" +└── Main Menu +``` + +Voice control does have some major advantages as we can allow the user to specify names too. For example: + +* "Start creating route 'To Tesco'" +* "Add waypoint 'Corner of Moor Road'" + +If the name is empty, we can either guess a good marker name or fall back to using "Waypoint X". + +#### Callback filtering +This sort of goes along with audio profiles, but it simply a way of getting the app to quieten down when there's too much to describe. Perhaps this should just be an audio profile e.g. "Quieter"? +``` +Callback filtering +├── Mute callbacks +├── Reduce callbacks +├── Increase callbacks +└── Main Menu +``` diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ad6d2b6a..b74a2028 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,7 @@ protobuf-plugin = "0.9.6" dagger-hilt = "2.57" devtools-ksp = "2.3.2" google-services = "4.4.3" -jetbrains-dokka = "2.0.0" +jetbrains-dokka = "2.2.0-Beta" firebase-crashlytics = "3.0.6" accompanistPermissions = "0.37.3" androidGpxParser = "2.3.1" From 1ab3fa9c2d2a690bf4c43e618c9f9da99f6eb1d5 Mon Sep 17 00:00:00 2001 From: davecraig <8530624+davecraig@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:58:51 +0000 Subject: [PATCH 17/18] Bump version to 0.3.12, version code 174 --- app/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 212c8432..8c0386fe 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -47,8 +47,8 @@ android { applicationId = "org.scottishtecharmy.soundscape" minSdk = 30 targetSdk = 35 - versionCode = 173 - versionName = "0.3.11" + versionCode = 174 + versionName = "0.3.12" // Maintaining this list means that we can exclude translations that aren't complete yet resourceConfigurations.addAll(listOf( From aaa90ef30772c0c344f1390d629dc1b1b4a7a6e3 Mon Sep 17 00:00:00 2001 From: davecraig <8530624+davecraig@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:09:15 +0000 Subject: [PATCH 18/18] New documentation screenshots --- docs/documentationScreens/homeScreen.png | Bin 399710 -> 399710 bytes .../homeScreenWithRoute.png | Bin 312845 -> 312845 bytes docs/documentationScreens/routeDetails.png | Bin 229702 -> 229702 bytes docs/documentationScreens/routeEdit.png | Bin 124145 -> 124145 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/documentationScreens/homeScreen.png b/docs/documentationScreens/homeScreen.png index bde723af3e4e61b925d262684ab2a1b109071572..f623f36c0b25e26d633f4a3892ace50e9c2e87f9 100644 GIT binary patch delta 189 zcmcb&N#foni3wTtOah#|s^48^GcYiymbgZgq$HN4S|t~y0x1R~10yqC17lqSqYy(g zD+5a_Lj!FC11kfA@N-@6C>nC}Q!>*kacfYUZ1fwbK?80>NoHPsvQH#I0f8*VS%74H|G8N-}d(i%Sx7C%)tsftzjw gG5rU}1eRtF#&!-y5M}~mW*}w(V%F^(jBH^N02C=SQUCw| diff --git a/docs/documentationScreens/homeScreenWithRoute.png b/docs/documentationScreens/homeScreenWithRoute.png index 679fd30440ac93cdee167d18029d6dc006f88a5b..449edc83d96af9469a45d117a893812e2cda4220 100644 GIT binary patch delta 181 zcmeC}5$^2~o{&|~B*4k5`rTzV0|SFRdP`(kYX@0Ff!9MFxE9N3NbXZ zGO)BVG|)CMure?RKiB1sq9HdwB{QuOw+6MzM!$g?G~hOrWag$8mn7y+e90{WH{BRw ax2dY5H#BoFwsSCoFcT0nZ|7iSVLb`}McXn% diff --git a/docs/documentationScreens/routeDetails.png b/docs/documentationScreens/routeDetails.png index 6930c4647a8585076ee8dd3185e229160cc3933b..406d7452f06dc1254efc913530fdd2c27ac0610e 100644 GIT binary patch delta 177 zcmX@s#CNQTZ$ef*lK>~*o`0f>3=9maC9V-ADTyViR>?)FK#IZ0z{pJ3z*yJ7D8$gr z%D~df&_LV3z{_K;wC=>1_lPz64!{5l*E!$tK_0oAjM#0U}UCiV61Ck7-DE_ zWn^GwWTK?80>NoHnC}Q!>*kacfYUZ1fwbK?80>NoHuNWk1`W6kC7HRY#U+Wk6JK(Rz)d%X Tm>yR@dqXn^<8}^4#+Q!)zMC?(