diff --git a/app/.gitignore b/app/.gitignore index 42afabfd2..56dadcc28 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -1 +1,3 @@ -/build \ No newline at end of file +/build +/cache-test +/cache-test2 diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 9bf2e70b1..8c0386fe0 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -47,8 +47,8 @@ android { applicationId = "org.scottishtecharmy.soundscape" minSdk = 30 targetSdk = 35 - versionCode = 172 - versionName = "0.3.10" + versionCode = 174 + versionName = "0.3.12" // Maintaining this list means that we can exclude translations that aren't complete yet resourceConfigurations.addAll(listOf( @@ -367,3 +367,11 @@ dependencies { testImplementation(libs.json) } + +dokka { + dokkaSourceSets.configureEach { + if (name == "main") { + suppress.set(true) + } + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 4d81c65a5..38924d72c 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -13,6 +13,8 @@ + + @@ -146,6 +148,9 @@ + + + \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt index a4750dfe6..f8fde362d 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/MainActivity.kt @@ -96,6 +96,10 @@ class MainActivity : AppCompatActivity() { checkAndRequestLocationPermissions() } + // Microphone permission for voice commands — best-effort; if denied, voice commands are silently skipped + private val micPermissionLauncher = + registerForActivityResult(ActivityResultContracts.RequestPermission()) { /* no-op */ } + // we need location permission to be able to start the service private val locationPermissionRequest = registerForActivityResult( ActivityResultContracts.RequestMultiplePermissions() @@ -164,6 +168,12 @@ class MainActivity : AppCompatActivity() { hintsEnabled = preferences.getBoolean(HINTS_KEY, HINTS_DEFAULT) ) } + + MEDIA_CONTROLS_MODE_KEY -> { + val mode = preferences.getString(MEDIA_CONTROLS_MODE_KEY, MEDIA_CONTROLS_MODE_DEFAULT)!! + Log.e(TAG, "mediaControlsMode $mode") + soundscapeServiceConnection.soundscapeService?.updateMediaControls(mode) + } } } @@ -746,6 +756,11 @@ class MainActivity : AppCompatActivity() { Log.e(TAG, "startSoundscapeService") val serviceIntent = Intent(this, SoundscapeService::class.java) startForegroundService(serviceIntent) + // Request microphone permission for voice commands (best-effort) + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != android.content.pm.PackageManager.PERMISSION_GRANTED) { + micPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } } companion object { @@ -796,6 +811,8 @@ class MainActivity : AppCompatActivity() { const val GEOCODER_MODE_KEY = "GeocoderMode" const val LAST_SPLASH_RELEASE_DEFAULT = "" const val LAST_SPLASH_RELEASE_KEY = "LastNewRelease" + const val MEDIA_CONTROLS_MODE_DEFAULT = "Original" + const val MEDIA_CONTROLS_MODE_KEY = "MediaControlsMode" const val FIRST_LAUNCH_KEY = "FirstLaunch" const val AUDIO_TOUR_SHOWN_KEY = "AudioTourShown" diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt index 0326813fd..8d4042427 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/SoundscapeServiceConnection.kt @@ -18,6 +18,7 @@ import org.scottishtecharmy.soundscape.services.BeaconState import org.scottishtecharmy.soundscape.services.RoutePlayerState import org.scottishtecharmy.soundscape.services.SoundscapeBinder import org.scottishtecharmy.soundscape.services.SoundscapeService +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import javax.inject.Inject @ActivityRetainedScoped @@ -48,13 +49,17 @@ class SoundscapeServiceConnection @Inject constructor() { return soundscapeService?.gridStateFlow } + fun getVoiceCommandStateFlow(): StateFlow? { + return soundscapeService?.voiceCommandStateFlow + } + fun setStreetPreviewMode(on : Boolean, location: LngLatAlt? = null) { Log.d(TAG, "setStreetPreviewMode $on") soundscapeService?.setStreetPreviewMode(on, location) } fun routeStart(routeId: Long) { - soundscapeService?.routeStart(routeId) + soundscapeService?.routeStartById(routeId) } fun startBeacon(location: LngLatAlt, name: String) { diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt index 139855156..f8ba87552 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt @@ -438,7 +438,7 @@ class GeoEngine { // So long as the AudioEngine is not already busy, run any auto callouts that we // need. Auto Callouts use the direction of travel if there is one, otherwise // falling back to use the phone direction. - if((!soundscapeService.isAudioEngineBusy() || streetPreview.running) && !autoCalloutDisabled) { + if((!soundscapeService.isAudioEngineBusy() || streetPreview.running) && !autoCalloutDisabled && !soundscapeService.menuActive) { val callout = autoCallout.updateLocation( getCurrentUserGeometry(UserGeometry.HeadingMode.CourseAuto), diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt index 483910fd9..f7c3e0e0d 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/Home.kt @@ -230,7 +230,8 @@ fun Home( goToAppSettings = goToAppSettings, fullscreenMap = fullscreenMap, permissionsRequired = permissionsRequired, - showMap = showMap + showMap = showMap, + voiceCommandListening = state.voiceCommandListening ) } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt index 0b5ad0aa4..86be79cf5 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/home/HomeContent.kt @@ -1,6 +1,7 @@ package org.scottishtecharmy.soundscape.screens.home.home import android.content.Context +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -8,6 +9,7 @@ import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.defaultMinSize +import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding @@ -160,14 +162,16 @@ fun HomeContent( goToAppSettings: (Context) -> Unit, fullscreenMap: MutableState, permissionsRequired: Boolean, - showMap: Boolean) { + showMap: Boolean, + voiceCommandListening: Boolean = false) { val context = LocalContext.current val coroutineScope = rememberCoroutineScope() var fetchingLocation by remember { mutableStateOf(false) } + Box(modifier = modifier.fillMaxSize()) { Column( verticalArrangement = Arrangement.spacedBy(spacing.small), - modifier = modifier + modifier = Modifier.fillMaxSize() ) { if (streetPreviewState.enabled != StreetPreviewEnabled.OFF) { StreetPreview(streetPreviewState, streetPreviewFunctions) @@ -390,6 +394,25 @@ fun HomeContent( } } } + if (voiceCommandListening) { + Box( + contentAlignment = Alignment.BottomCenter, + modifier = Modifier.fillMaxSize() + ) { + Text( + text = stringResource(R.string.voice_cmd_listening), + style = MaterialTheme.typography.bodyLarge, + color = MaterialTheme.colorScheme.onSurface, + modifier = Modifier + .fillMaxWidth() + .background(MaterialTheme.colorScheme.surface.copy(alpha = 0.85f)) + .padding(vertical = spacing.small) + .semantics { liveRegion = LiveRegionMode.Assertive }, + textAlign = TextAlign.Center + ) + } + } + } // end Box } @Preview diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt index 3893c4b38..78adb1fd8 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/home/settings/Settings.kt @@ -258,6 +258,17 @@ fun Settings( "Offline" ) + val mediaControlsDescriptions = listOf( + stringResource(R.string.settings_media_controls_original), + stringResource(R.string.settings_media_controls_voice_command), + stringResource(R.string.settings_media_controls_audio_menu), + ) + val mediaControlsValues = listOf( + "Original", + "VoiceControl", + "AudioMenu", + ) + if (showConfirmationDialog.value) { AlertDialog( onDismissRequest = { showConfirmationDialog.value = false }, @@ -683,6 +694,42 @@ fun Settings( } } + // Media control section + item(key = "header_media_control") { + ExpandableSectionHeader( + title = stringResource(R.string.menu_media_controls), + expanded = expandedSection.value == "media_controls", + onToggle = { expandedSection.value = if (expandedSection.value == "media_controls") null else "media_controls" }, + textColor = textColor + ) + } + if (expandedSection.value == "media_controls") { + listPreference( + key = MainActivity.MEDIA_CONTROLS_MODE_KEY, + defaultValue = MainActivity.MEDIA_CONTROLS_MODE_DEFAULT, + values = mediaControlsValues, + modifier = expandedSectionModifier, + title = { + SettingDetails( + R.string.settings_section_media_controls, + R.string.settings_section_media_controls_description, + textColor + ) + }, + item = { value, currentValue, onClick -> + ListPreferenceItem(mediaControlsDescriptions[mediaControlsValues.indexOf(value)], value, currentValue, onClick, mediaControlsValues.indexOf(value), mediaControlsValues.size) + }, + summary = { + Text( + text = mediaControlsDescriptions[mediaControlsValues.indexOf(it)], + color = textColor, + style = MaterialTheme.typography.bodyLarge + ) + }, + ) + } + + // Debug Section item(key = "header_debug") { ExpandableSectionHeader( @@ -800,6 +847,11 @@ fun SettingsPreviewLanguage() { } @Preview @Composable +fun SettingsPreviewMediaControls() { + SettingsPreview("media_controls") +} +@Preview +@Composable fun SettingsPreviewDebug() { SettingsPreview("debug") } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt b/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt index b51c1f611..d1cdbe445 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/screens/onboarding/navigating/NavigatingScreen.kt @@ -19,6 +19,7 @@ import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.LocationOn +import androidx.compose.material.icons.rounded.Mic import androidx.compose.material.icons.rounded.Notifications import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme @@ -51,12 +52,14 @@ fun NavigatingScreen( arrayOf( Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_BACKGROUND_LOCATION, - Manifest.permission.POST_NOTIFICATIONS + Manifest.permission.POST_NOTIFICATIONS, + Manifest.permission.RECORD_AUDIO ) } else { arrayOf( Manifest.permission.ACCESS_FINE_LOCATION, - Manifest.permission.ACCESS_BACKGROUND_LOCATION + Manifest.permission.ACCESS_BACKGROUND_LOCATION, + Manifest.permission.RECORD_AUDIO ) } val multiplePermissionResultLauncher = rememberLauncherForActivityResult( @@ -135,7 +138,7 @@ fun Navigating( contentDescription = null, tint = MaterialTheme.colorScheme.onSurface ) - Spacer(modifier = Modifier.width(spacing.medium)) + Spacer(modifier = Modifier.width(spacing.extraSmall)) Column( modifier = Modifier.semantics(mergeDescendants = true) {}, ) { @@ -182,7 +185,33 @@ fun Navigating( ) } } - + } + Row( + modifier = Modifier + .mediumPadding() + .fillMaxWidth(), + ) + { + Icon( + Icons.Rounded.Mic, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurface + ) + Spacer(modifier = Modifier.width(spacing.extraSmall)) + Column( + modifier = Modifier.semantics(mergeDescendants = true) {}, + ) { + Text( + text = stringResource(R.string.first_launch_permissions_record_audio), + style = MaterialTheme.typography.titleMedium, + color = MaterialTheme.colorScheme.onSurface, + ) + Text( + text = stringResource(R.string.first_launch_permissions_required_for_voice_control), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface, + ) + } } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt deleted file mode 100644 index 61067709b..000000000 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeDummyMediaPlayer.kt +++ /dev/null @@ -1,36 +0,0 @@ -package org.scottishtecharmy.soundscape.services - -import android.os.Looper -import androidx.annotation.OptIn -import androidx.media3.common.DeviceInfo -import androidx.media3.common.Player -import androidx.media3.common.SimpleBasePlayer -import androidx.media3.common.util.UnstableApi - -@OptIn(UnstableApi::class) -class SoundscapeDummyMediaPlayer : SimpleBasePlayer(Looper.getMainLooper()) { - - override fun getState(): State { - - val commands : Player.Commands = Player.Commands.EMPTY - commands.buildUpon().addAll( - COMMAND_PLAY_PAUSE, - COMMAND_STOP, - COMMAND_SEEK_TO_NEXT, - COMMAND_SEEK_TO_PREVIOUS, - COMMAND_SEEK_FORWARD, - COMMAND_SEEK_BACK, - COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, - COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM) - - return State.Builder() - // Set which playback commands the player can handle - .setAvailableCommands(commands) - // Configure additional playback properties - .setPlayWhenReady(true, PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST) - .setCurrentMediaItemIndex(0) - .setContentPositionMs(0) - .setDeviceInfo(DeviceInfo.Builder(DeviceInfo.PLAYBACK_TYPE_LOCAL).build()) - .build() - } -} \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index a29266974..a5b04cc1e 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -1,9 +1,11 @@ package org.scottishtecharmy.soundscape.services +import android.Manifest import android.content.Context import android.content.res.Configuration import android.annotation.SuppressLint import android.app.ForegroundServiceStartNotAllowedException +import android.content.pm.PackageManager import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager @@ -23,9 +25,11 @@ import android.widget.Toast import androidx.annotation.OptIn import androidx.core.app.NotificationCompat import androidx.core.app.ServiceCompat +import androidx.core.content.ContextCompat import androidx.media3.common.util.UnstableApi import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService +import androidx.preference.PreferenceManager import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -35,14 +39,22 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.flow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.scottishtecharmy.soundscape.MainActivity +import org.scottishtecharmy.soundscape.MainActivity.Companion.MEDIA_CONTROLS_MODE_DEFAULT +import org.scottishtecharmy.soundscape.MainActivity.Companion.MEDIA_CONTROLS_MODE_KEY import org.scottishtecharmy.soundscape.R import org.scottishtecharmy.soundscape.audio.AudioType import org.scottishtecharmy.soundscape.audio.NativeAudioEngine +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_CALLOUTS_ON +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_ENTER +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_EXIT import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase +import org.scottishtecharmy.soundscape.database.local.model.MarkerEntity +import org.scottishtecharmy.soundscape.database.local.model.RouteEntity import org.scottishtecharmy.soundscape.geoengine.GeoEngine import org.scottishtecharmy.soundscape.geoengine.GridState import org.scottishtecharmy.soundscape.geoengine.StreetPreviewChoice @@ -60,6 +72,15 @@ import org.scottishtecharmy.soundscape.locationprovider.DirectionProvider import org.scottishtecharmy.soundscape.locationprovider.LocationProvider import org.scottishtecharmy.soundscape.locationprovider.StaticLocationProvider import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.mediacontrol.AudioMenu +import org.scottishtecharmy.soundscape.services.mediacontrol.AudioMenuMediaControls +import org.scottishtecharmy.soundscape.services.mediacontrol.MediaControlTarget +import org.scottishtecharmy.soundscape.services.mediacontrol.OriginalMediaControls +import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeDummyMediaPlayer +import org.scottishtecharmy.soundscape.services.mediacontrol.SoundscapeMediaSessionCallback +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandManager +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandMediaControls +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import org.scottishtecharmy.soundscape.utils.Analytics import org.scottishtecharmy.soundscape.utils.getCurrentLocale import kotlin.time.Duration @@ -93,6 +114,12 @@ class SoundscapeService : MediaSessionService() { var audioEngine = NativeAudioEngine(this) private var audioBeacon: Long = 0 + // Audio menu (navigated via media buttons when no route is active) + var audioMenu : AudioMenu? = null + + /** True while the user is actively navigating the audio menu. Suppresses auto callouts. */ + var menuActive: Boolean = false + // Audio focus private lateinit var audioManager: AudioManager private var audioFocusRequest: AudioFocusRequest? = null @@ -148,9 +175,16 @@ class SoundscapeService : MediaSessionService() { private val _gridStateFlow = MutableStateFlow(null) var gridStateFlow: StateFlow = _gridStateFlow + // Voice command manager — only initialized when RECORD_AUDIO permission is granted + private var voiceCommandManager: VoiceCommandManager? = null + val voiceCommandStateFlow: StateFlow + get() = voiceCommandManager?.state ?: MutableStateFlow(VoiceCommandState.Idle) + // Media control button code private var mediaSession: MediaSession? = null - private val mediaPlayer = SoundscapeDummyMediaPlayer() + + private var mediaControlsTarget : MediaControlTarget = OriginalMediaControls(this) + private val mediaPlayer = SoundscapeDummyMediaPlayer { mediaControlsTarget } var running: Boolean = false var started: Boolean = false @@ -196,10 +230,6 @@ class SoundscapeService : MediaSessionService() { locationProvider.start(this) directionProvider.start(audioEngine, locationProvider) - val configLocale = getCurrentLocale() - val configuration = Configuration(applicationContext.resources.configuration) - configuration.setLocale(configLocale) - localizedContext = applicationContext.createConfigurationContext(configuration) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) } @@ -234,6 +264,7 @@ class SoundscapeService : MediaSessionService() { val configuration = Configuration(applicationContext.resources.configuration) configuration.setLocale(configLocale) localizedContext = applicationContext.createConfigurationContext(configuration) + voiceCommandManager?.updateContext(localizedContext) geoEngine.start(application, locationProvider, directionProvider, this, localizedContext) started = true } @@ -259,7 +290,7 @@ class SoundscapeService : MediaSessionService() { audioEngine.initialize(applicationContext) audioManager = getSystemService(AUDIO_SERVICE) as AudioManager - + audioMenu = AudioMenu(this, application) routePlayer = RoutePlayer(this, applicationContext) if(hasPlayServices(this)) { @@ -273,16 +304,60 @@ class SoundscapeService : MediaSessionService() { // create new RealmDB or open existing startRealms(applicationContext) + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + == PackageManager.PERMISSION_GRANTED) { + voiceCommandManager = VoiceCommandManager( + service = this, + onError = { } + ) + } + + // Update the media controls mode + val sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) + val mode = sharedPreferences.getString(MEDIA_CONTROLS_MODE_KEY, MEDIA_CONTROLS_MODE_DEFAULT)!! + updateMediaControls(mode) + + // Keep biasing strings up to date whenever markers or routes change + val dao = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao() + coroutineScope.launch { + dao.getAllMarkersFlow().collect { markers -> + voiceCommandManager?.updateMarkers(markers) + } + } + coroutineScope.launch { + dao.getAllRoutesFlow().collect { routes -> + voiceCommandManager?.updateRoutes(routes) + } + } + mediaSession = MediaSession.Builder(this, mediaPlayer) - .setCallback(SoundscapeMediaSessionCallback(this)) + .setId("org.scottishtecharmy.soundscape") + .setCallback(SoundscapeMediaSessionCallback { mediaControlsTarget }) .build() } } + fun updateMediaControls(target: String) { + val hasRecordAudio = ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + mediaControlsTarget = when (target) { + "VoiceControl" if hasRecordAudio -> { + voiceCommandManager?.initialize() + VoiceCommandMediaControls(this) + } + "VoiceControl" -> AudioMenuMediaControls(audioMenu) + "AudioMenu" -> AudioMenuMediaControls(audioMenu) + "Original" -> OriginalMediaControls(this) + else -> OriginalMediaControls(this) + } + } + override fun onTaskRemoved(rootIntent: Intent?) { Log.d(TAG, "onTaskRemoved for service - ignoring, as we want to keep running") } override fun onDestroy() { + suppressionJob?.cancel() + // If _mediaSession is not null, run the following block mediaSession?.run { // Release the player @@ -295,6 +370,7 @@ class SoundscapeService : MediaSessionService() { super.onDestroy() Log.d(TAG, "onDestroy") + audioMenu?.destroy() audioEngine.destroyBeacon(audioBeacon) audioBeacon = 0 audioEngine.destroy() @@ -313,6 +389,8 @@ class SoundscapeService : MediaSessionService() { wakeLock?.let { if (it.isHeld) it.release() } wakeLock = null + voiceCommandManager?.destroy() + // Clear service reference in binder so that it can be garbage collected binder?.reset() } @@ -505,6 +583,36 @@ class SoundscapeService : MediaSessionService() { } } + fun triggerVoiceCommand() { + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != PackageManager.PERMISSION_GRANTED) return + + if (!requestAudioFocus()) { + Log.w(TAG, "speakText: Could not get audio focus. Aborting callouts.") + return + } + + // Stop callbacks whilst we handle voice commands + callbackHoldOff() + + val ctx = if (::localizedContext.isInitialized) localizedContext else this + + // Play earcon as feedback to indicate that we're starting to listen. + speak2dText("", true, EARCON_CALLOUTS_ON) + + // Wait for the TTS to finish before opening the mic, otherwise we lose audio focus before + // the icon completes. + coroutineScope.launch { + val deadline = System.currentTimeMillis() + 1000L + while (isAudioEngineBusy() && System.currentTimeMillis() < deadline) { + delay(20) + } + withContext(Dispatchers.Main) { + voiceCommandManager?.startListening() + } + } + } + suspend fun searchResult(searchString: String): List? { return geoEngine.searchResult(searchString) } @@ -516,7 +624,7 @@ class SoundscapeService : MediaSessionService() { fun startBeacon(location: LngLatAlt, name: String) { routePlayer.startBeacon(location, name) } - fun routeStart(routeId: Long) { + fun routeStartById(routeId: Long) { routePlayer.startRoute(routeId) } fun routeStop() { @@ -542,6 +650,46 @@ class SoundscapeService : MediaSessionService() { } return false } + fun routeListRoutes() { + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val routes = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllRoutes() + if (routes.isEmpty()) + speak2dText(ctx.getString(R.string.voice_cmd_no_routes)) + else { + val names = routes.joinToString(". ") { it.name } + speak2dText(ctx.getString(R.string.voice_cmd_routes_list) + names) + } + } + } + + fun routeStart(route: RouteEntity) { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + speak2dText(ctx.getString(R.string.voice_cmd_starting_route).format(route.name)) + routeStartById(route.routeId) + } + + fun routeListMarkers() { + coroutineScope.launch { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + val markers = MarkersAndRoutesDatabase.getMarkersInstance(applicationContext).routeDao().getAllMarkers() + if (markers.isEmpty()) { + speak2dText(ctx.getString(R.string.voice_cmd_no_markers)) + } else { + val names = markers.joinToString(". ") { it.name } + speak2dText(ctx.getString(R.string.voice_cmd_markers_list) + names) + } + } + } + + fun markerStart(marker: MarkerEntity) { + val ctx = if (::localizedContext.isInitialized) localizedContext else this@SoundscapeService + speak2dText(ctx.getString(R.string.voice_cmd_starting_beacon_at_marker).format(marker.name)) + + val location = LngLatAlt(marker.longitude, marker.latitude) + startBeacon(location, marker.name) + } + /** * isAudioEngineBusy returns true if there is more than one entry in the * audio engine queue. The queue consists of earcons and text-to-speech. @@ -559,24 +707,37 @@ class SoundscapeService : MediaSessionService() { heading: Double = Double.NaN) { if (!requestAudioFocus()) { - Log.w(TAG, "speakText: Could not get audio focus. Aborting callouts.") + Log.w(TAG, "speakText: Could not get audio focus.") return } Log.d(TAG, "speakText $text") audioEngine.createTextToSpeech(text, type, latitude, longitude, heading) } + fun speak2dText(text: String, clearQueue: Boolean = false, earcon: String? = null) { + if (!requestAudioFocus()) { + Log.w(TAG, "speak2dText: Could not get audio focus.") + return + } + if(clearQueue) + audioEngine.clearTextToSpeechQueue() + if(earcon != null) { + audioEngine.createEarcon(earcon, AudioType.STANDARD) + } + if(text.isNotEmpty()) + audioEngine.createTextToSpeech(text, AudioType.STANDARD) + } fun speakCallout(callout: TrackedCallout?, addModeEarcon: Boolean) { if(callout == null) return if (!requestAudioFocus()) { - Log.w(TAG, "SpeakCallout: Could not get audio focus. Aborting callouts.") + Log.w(TAG, "SpeakCallout: Could not get audio focus.") return } - if(addModeEarcon) audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_ENTER, AudioType.STANDARD) + if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_ENTER, AudioType.STANDARD) for(result in callout.positionedStrings) { if(result.location == null) { var type = result.type @@ -604,7 +765,7 @@ class SoundscapeService : MediaSessionService() { ) } } - if(addModeEarcon) audioEngine.createEarcon(NativeAudioEngine.EARCON_MODE_EXIT, AudioType.STANDARD) + if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_EXIT, AudioType.STANDARD) callout.calloutHistory?.add(callout) callout.locationFilter?.update(callout.userGeometry) @@ -701,12 +862,30 @@ class SoundscapeService : MediaSessionService() { audioFocusGained = false } + /** + * Called on every menu interaction. Marks the menu as active (suppressing auto callouts) + * and resets the 10-second countdown after which auto callouts are re-enabled. + */ + /** Cancels pending re-enable of auto callouts and restarts the 10-second countdown. */ + private var suppressionJob: Job? = null + private val scope = CoroutineScope(Dispatchers.Default) + fun callbackHoldOff() { + menuActive = true + suppressionJob?.cancel() + suppressionJob = scope.launch { + delay(CALLOUT_SUPPRESS_TIMEOUT_MS) + menuActive = false + } + } + companion object { private const val TAG = "SoundscapeService" // Secondary "service" every n seconds private val TICKER_PERIOD_SECONDS = 3600.seconds + private const val CALLOUT_SUPPRESS_TIMEOUT_MS = 8_000L + private const val CHANNEL_ID = "SoundscapeService_channel_01" private const val NOTIFICATION_CHANNEL_NAME = "Soundscape_SoundscapeService" private const val NOTIFICATION_ID = 100000 diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt new file mode 100644 index 000000000..6db49c0e5 --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/AudioMenu.kt @@ -0,0 +1,266 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import android.content.Context +import android.content.res.Configuration +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.scottishtecharmy.soundscape.R +import org.scottishtecharmy.soundscape.audio.AudioType +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_ENTER +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_MODE_EXIT +import org.scottishtecharmy.soundscape.database.local.MarkersAndRoutesDatabase +import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.services.SoundscapeService +import org.scottishtecharmy.soundscape.utils.getCurrentLocale + +/** + * AudioMenu provides a hierarchical, navigable audio menu controlled by media buttons. + * + * - NEXT : advance to next item at the current level (wraps at end) + * - PREVIOUS: go back one item (wraps at end) + * - SELECT : enter a sub-menu, or execute a leaf action + * + * There is no inactivity timeout — the current position is remembered until changed. + */ +class AudioMenu( + private val service: SoundscapeService, + private val application: Context) { + + // ── Menu item types ─────────────────────────────────────────────────────── + + sealed class MenuItem { + abstract val label: String + + /** A leaf node that executes an action when selected. */ + data class Action( + override val label: String, + val action: () -> Unit + ) : MenuItem() + + /** A sub-menu with a fixed list of children. */ + data class Submenu( + override val label: String, + val children: List + ) : MenuItem() + + /** + * A sub-menu whose children are loaded lazily when the user enters it. + * Useful for content that may change at runtime (e.g. saved routes). + */ + data class DynamicSubmenu( + override val label: String, + val childrenProvider: suspend () -> List + ) : MenuItem() + } + + // ── Navigation state ────────────────────────────────────────────────────── + private data class MenuLevel(val items: List, var currentIndex: Int) + + /** Stack of levels; the root is at index 0, deeper sub-menus higher up. */ + private val menuStack = ArrayDeque() + + private val scope = CoroutineScope(Dispatchers.Default) + + val localizedContext: Context + init { + val configLocale = getCurrentLocale() + val configuration = Configuration(application.applicationContext.resources.configuration) + configuration.setLocale(configLocale) + localizedContext = application.applicationContext.createConfigurationContext(configuration) + menuStack.addLast(MenuLevel(buildRootMenu(), 0)) + } + + // ── Public navigation API ───────────────────────────────────────────────── + + fun next() { + service.callbackHoldOff() + val label = synchronized(this) { + val level = menuStack.last() + level.currentIndex = (level.currentIndex + 1) % level.items.size + level.items[level.currentIndex].label + } + service.speak2dText(label, true) + } + + fun previous() { + service.callbackHoldOff() + val label = synchronized(this) { + val level = menuStack.last() + level.currentIndex = + if(level.currentIndex == 0) + level.items.size - 1 + else + level.currentIndex - 1 + level.items[level.currentIndex].label + } + service.speak2dText(label, true) + } + + fun select() { + service.callbackHoldOff() + val item = synchronized(this) { menuStack.last().let { it.items[it.currentIndex] } } + when (item) { + is MenuItem.Action -> { + item.action() + } + is MenuItem.Submenu -> { + val firstLabel = synchronized(this) { + menuStack.addLast(MenuLevel(item.children, 0)) + item.children[0].label + } + service.speak2dText(firstLabel, true, EARCON_MODE_ENTER) + } + is MenuItem.DynamicSubmenu -> loadAndEnter(item) + } + } + + // ── Main-menu escape ────────────────────────────────────────────────────── + + /** + * Returns a "Main Menu" Action item that, when selected, pops the entire + * stack back to the root and announces the first root-level item. + * Appended as the last child of every sub-menu. + */ + private fun mainMenuAction(): MenuItem.Action = + MenuItem.Action(localizedContext.getString(R.string.menu_main_menu)) { + resetToRoot() + } + + private fun resetToRoot() { + val firstRootLabel = synchronized(this) { + while (menuStack.size > 1) menuStack.removeLast() + menuStack.last().currentIndex = 0 + menuStack.last().items[0].label + } + if (!service.requestAudioFocus()) { + Log.w(TAG, "resetToRoot: could not get audio focus") + return + } + service.speak2dText(firstRootLabel, true, EARCON_MODE_EXIT) + } + + // ── Audio helpers ───────────────────────────────────────────────────────── + private fun loadAndEnter(item: MenuItem.DynamicSubmenu) { + scope.launch { + val children = item.childrenProvider() + if (children.size <= 1) { + service.speakText(localizedContext.getString(R.string.menu_no_routes), AudioType.STANDARD) + } else { + val firstLabel = synchronized(this@AudioMenu) { + menuStack.addLast(MenuLevel(children, 0)) + children[0].label + } + service.speak2dText(firstLabel, true, EARCON_MODE_ENTER) + } + } + } + + private fun audioProfileAction(@androidx.annotation.StringRes id: Int, profile: String): MenuItem.Action { + val label = localizedContext.getString(id) + return MenuItem.Action(label) { + applyAudioProfile(profile) + service.speak2dText(label) + } + } + + // ── Menu definition ─────────────────────────────────────────────────────── + + private fun buildRootMenu(): List = listOf( + + MenuItem.Submenu( + label = localizedContext.getString(R.string.callouts_panel_title), + children = listOf( + MenuItem.Action(localizedContext.getString(R.string.directions_my_location)) { + service.myLocation() + }, + MenuItem.Action(localizedContext.getString(R.string.help_orient_page_title)) { + service.whatsAroundMe() + }, + MenuItem.Action(localizedContext.getString(R.string.help_explore_page_title)) { + service.aheadOfMe() + }, + MenuItem.Action(localizedContext.getString(R.string.callouts_nearby_markers)) { + service.nearbyMarkers() + }, + mainMenuAction(), + ) + ), + + MenuItem.Submenu( + label = localizedContext.getString(R.string.menu_route), + children = listOf( + MenuItem.Action(localizedContext.getString(R.string.menu_route_next_waypoint)) { + service.routeSkipNext() + }, + MenuItem.Action(localizedContext.getString(R.string.menu_route_previous_waypoint)) { + service.routeSkipPrevious() + }, + MenuItem.Action(localizedContext.getString(R.string.beacon_action_mute_beacon)) { + service.routeMute() + }, + MenuItem.Action(localizedContext.getString(R.string.route_detail_action_stop_route)) { + service.routeStop() + }, + mainMenuAction(), + ) + ), + + MenuItem.DynamicSubmenu( + label = localizedContext.getString(R.string.route_detail_action_start_route), + childrenProvider = { loadRouteMenuItems() } + ), + + MenuItem.DynamicSubmenu( + label = localizedContext.getString(R.string.location_detail_action_beacon), + childrenProvider = { loadMarkerMenuItems() } + ), + ) + + // ── Feature implementations ─────────────────────────────────────────────── + + private suspend fun loadRouteMenuItems(): List = + withContext(Dispatchers.IO) { + val db = MarkersAndRoutesDatabase.getMarkersInstance(service) + db.routeDao().getAllRoutes().map { route -> + MenuItem.Action(route.name) { service.routeStartById(route.routeId) } + } + mainMenuAction() + } + + private suspend fun loadMarkerMenuItems(): List = + withContext(Dispatchers.IO) { + val db = MarkersAndRoutesDatabase.getMarkersInstance(service) + db.routeDao().getAllMarkers().map { marker -> + MenuItem.Action(marker.name) { + val location = LngLatAlt(marker.longitude, marker.latitude) + service.startBeacon(location, marker.name) + } + } + mainMenuAction() + } + + /** + */ + private fun applyAudioProfile(profileName: String) { + when (profileName) { + "eating" -> {} + "shopping" -> {} + "navigating" -> {} + "roads_only" -> {} + "all" -> {} + } + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + fun destroy() { + service.menuActive = false + scope.cancel() + } + + companion object { + private const val TAG = "AudioMenu" + } +} diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt new file mode 100644 index 000000000..77e361818 --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/MediaControlTarget.kt @@ -0,0 +1,65 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import org.scottishtecharmy.soundscape.services.SoundscapeService + +// An interface that encapsulates the various media control buttons +interface MediaControlTarget { + + fun onPlayPause() : Boolean + fun onNext() : Boolean + fun onPrevious() : Boolean +} + +// This is how the iOS Soundscape media controls behaved +class OriginalMediaControls(val service: SoundscapeService) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + service.routeMute() + return true + } + + override fun onNext() : Boolean { + if (!service.routeSkipNext()) { + // If there's no route playing, callout My Location. + service.myLocation() + } + return true + } + + override fun onPrevious() : Boolean { + if(!service.routeSkipPrevious()) { + // If there's no route playing, callout around me + service.whatsAroundMe() + } + return true + } +} + +// This is how the voice command media controls work +class VoiceCommandMediaControls(val service: SoundscapeService) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + service.triggerVoiceCommand() + return true + } + override fun onNext() : Boolean { return false } + override fun onPrevious() : Boolean { return false } +} + +// This is how the menu navigation media controls work +class AudioMenuMediaControls(val audioMenu: AudioMenu?) : MediaControlTarget { + + override fun onPlayPause() : Boolean { + audioMenu?.select() + return true + } + override fun onNext() : Boolean { + audioMenu?.next() + return true + } + override fun onPrevious() : Boolean { + audioMenu?.previous() + return true + } +} + diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt new file mode 100644 index 000000000..5efd95b07 --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeDummyMediaPlayer.kt @@ -0,0 +1,69 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import android.os.Looper +import androidx.annotation.OptIn +import androidx.media3.common.DeviceInfo +import androidx.media3.common.MediaItem +import androidx.media3.common.MediaMetadata +import androidx.media3.common.Player +import androidx.media3.common.SimpleBasePlayer +import androidx.media3.common.util.UnstableApi +import com.google.common.util.concurrent.Futures +import com.google.common.util.concurrent.ListenableFuture + +@OptIn(UnstableApi::class) +class SoundscapeDummyMediaPlayer( + private val getTarget: () -> MediaControlTarget +) : SimpleBasePlayer(Looper.getMainLooper()) { + + override fun getState(): State { + + val commands = Player.Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_STOP, + COMMAND_SEEK_TO_NEXT, + COMMAND_SEEK_TO_PREVIOUS, + COMMAND_SEEK_FORWARD, + COMMAND_SEEK_BACK, + COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, + COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM) + .build() + + val playlist = listOf( + MediaItemData.Builder("soundscape_item") + .setMediaItem( + MediaItem.Builder() + .setMediaId("soundscape_item") + .setMediaMetadata( + MediaMetadata.Builder() + .setTitle("Soundscape") + .setArtist("Scottish Tech Army") + .build() + ) + .build() + ) + .build() + ) + + return State.Builder() + // Set which playback commands the player can handle + .setAvailableCommands(commands) + // Configure additional playback properties + .setPlayWhenReady(true, PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST) + .setPlaylist(playlist) + .setCurrentMediaItemIndex(0) + .setContentPositionMs(0) + .setDeviceInfo(DeviceInfo.Builder(DeviceInfo.PLAYBACK_TYPE_LOCAL).build()) + .setPlaybackState(STATE_READY) + .build() + } + + override fun handleSetPlayWhenReady(playWhenReady: Boolean): ListenableFuture<*> { + getTarget().onPlayPause() + return Futures.immediateVoidFuture() + } + + override fun handleStop(): ListenableFuture<*> = + Futures.immediateVoidFuture() +} \ No newline at end of file diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt similarity index 70% rename from app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt rename to app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt index 4ee332fa0..41d4dd20d 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeMediaSessionCallback.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/SoundscapeMediaSessionCallback.kt @@ -1,4 +1,4 @@ -package org.scottishtecharmy.soundscape.services +package org.scottishtecharmy.soundscape.services.mediacontrol import android.content.Intent import android.util.Log @@ -12,7 +12,9 @@ import androidx.media3.session.MediaSession /** This callback class handles media button events generated by bluetooth headphones etc. * These are then mapped to specific Soundscape features. */ -class SoundscapeMediaSessionCallback(val service : SoundscapeService): +class SoundscapeMediaSessionCallback( + private val getTarget: () -> MediaControlTarget +): MediaSession.Callback { // Configure commands available to the controller in onConnect() @OptIn(UnstableApi::class) @@ -40,54 +42,46 @@ class SoundscapeMediaSessionCallback(val service : SoundscapeService): KeyEvent::class.java ) + // Ignore key-up events to prevent double-firing + if (keyEvent?.action != KeyEvent.ACTION_DOWN) return false + // So far I've only seen KEYCODE_MEDIA_PLAY_PAUSE, KEYCODE_MEDIA_PREVIOUS and // KEYCODE_MEDIA_NEXT, though that may be specific to my phone. The only event actually // handled for now is KEYCODE_MEDIA_NEXT. - // The behaviour of the media buttons changes when a route is being played back. In that - // case the buttons map to next/previous waypoint and muting audio. + // TODO: + // The behaviour of the media buttons changes when a route is being played back. In that + // case the buttons map to next/previous waypoint and muting audio. This currently doesn't + // play nicely with the audioMenu, more work is required. keyEvent?.let { event -> val decodedKey = when(event.keyCode) { KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { // ⏯ Play/Pause: Mute any current callouts and if the audio beacon is set, toggle the beacon audio. - service.routeMute() + getTarget().onPlayPause() "Play/Pause" } KeyEvent.KEYCODE_MEDIA_SKIP_FORWARD -> { // ⏩ Skip Forward - if(!service.routeSkipNext()) { - // If there's no route playing, toggle auto callouts. - service.toggleAutoCallouts() - } + getTarget().onNext() "Skip forward" } KeyEvent.KEYCODE_MEDIA_NEXT -> { // ⏭ Next - if(!service.routeSkipNext()) { - // If there's no route playing, callout My Location. - service.myLocation() - } + getTarget().onNext() "Next" } KeyEvent.KEYCODE_MEDIA_PREVIOUS -> { // ⏮ Previous - if(!service.routeSkipPrevious()) { - // TODO: : Repeat last callout. - } - "Previous" + getTarget().onPrevious() } KeyEvent.KEYCODE_MEDIA_SKIP_BACKWARD -> { // ⏪ Skip Backward - if(!service.routeSkipPrevious()) { - // If there's no route playing, callout around me - service.whatsAroundMe() - } + getTarget().onPrevious() "Skip backward" } - KeyEvent.KEYCODE_MEDIA_PLAY -> "Play" KeyEvent.KEYCODE_MEDIA_FAST_FORWARD -> "Fast forward" KeyEvent.KEYCODE_MEDIA_REWIND -> "Rewind" else -> return false diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt new file mode 100644 index 000000000..43bb12e1f --- /dev/null +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/mediacontrol/VoiceCommandManager.kt @@ -0,0 +1,289 @@ +package org.scottishtecharmy.soundscape.services.mediacontrol + +import android.content.Context +import android.content.Intent +import android.os.Build +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognitionSupport +import android.speech.RecognitionSupportCallback +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import androidx.core.content.ContextCompat +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.scottishtecharmy.soundscape.R +import org.scottishtecharmy.soundscape.audio.NativeAudioEngine.Companion.EARCON_CALLOUTS_OFF +import org.scottishtecharmy.soundscape.database.local.model.MarkerEntity +import org.scottishtecharmy.soundscape.database.local.model.RouteEntity +import org.scottishtecharmy.soundscape.services.SoundscapeService +import org.scottishtecharmy.soundscape.utils.fuzzyCompare +import org.scottishtecharmy.soundscape.utils.getCurrentLocale +import java.util.Locale + +sealed class VoiceCommandState { + object Idle : VoiceCommandState() + object Listening : VoiceCommandState() + object Error : VoiceCommandState() +} + +class VoiceCommandManager( + private val service: SoundscapeService, + private val onError: () -> Unit +) { + + private var context: Context = service + private var speechRecognizer: SpeechRecognizer? = null + private val _state = MutableStateFlow(VoiceCommandState.Idle) + val state: StateFlow = _state.asStateFlow() + @Volatile private var listOfRoutes: List = emptyList() + @Volatile private var listOfMarkers: List = emptyList() + // Language tag validated against the recognizer's supported list; set by initialize(). + private var cachedLanguage: String? = null + + /** Call this whenever SoundscapeService updates its localizedContext. */ + fun updateContext(newContext: Context) { + context = newContext + } + + fun updateRoutes(routes: List) { + listOfRoutes = routes + } + fun updateMarkers(markers: List) { + listOfMarkers = markers + } + + // Must be called on the main thread when switching to VoiceControl mode. + // Creates the SpeechRecognizer and pre-fetches the supported language list so that + // startListening() can start immediately without an extra round-trip. + fun initialize() { + destroyRecognizer() + cachedLanguage = null + + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + println("Recognition is unavailable") + onError() + return + } + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) + speechRecognizer?.setRecognitionListener(listener) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val probeIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + } + speechRecognizer?.checkRecognitionSupport( + probeIntent, + ContextCompat.getMainExecutor(context), + object : RecognitionSupportCallback { + override fun onSupportResult(recognitionSupport: RecognitionSupport) { + val supported = recognitionSupport.installedOnDeviceLanguages + + recognitionSupport.onlineLanguages + supported.forEach { println("Supported language: $it") } + cachedLanguage = pickBestLanguage(getCurrentLocale(), supported) + println("cachedLanguage $cachedLanguage") + } + + override fun onError(error: Int) { + // Support query failed — cachedLanguage stays null (no EXTRA_LANGUAGE). + } + } + ) + } + } + + // Must be called on the main thread (satisfied: service is on main thread) + fun startListening() { + if (_state.value is VoiceCommandState.Listening) return + speechRecognizer?.startListening(buildRecognitionIntent(cachedLanguage)) + } + + /** + * Choose the best BCP-47 language tag from [supportedTags] for the given [locale]. + * Prefers an exact match (language + country); falls back to any tag with the same language + * code; returns null if nothing matches. + */ + private fun pickBestLanguage(locale: Locale, supportedTags: List): String? { + + val tag = locale.toLanguageTag() + println("pickBestLanguage for $tag") + if (tag in supportedTags) return tag + val langCode = locale.language + return supportedTags.firstOrNull { Locale.forLanguageTag(it).language == langCode } + } + + private fun buildRecognitionIntent(language: String?): Intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + if (language != null) putExtra(RecognizerIntent.EXTRA_LANGUAGE, language) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val biasingStrings = ArrayList() + simpleCommands.forEach { biasingStrings.add(context.getString(it.stringId)) } + val markers = listOfMarkers + val routes = listOfRoutes + for(marker in markers) + biasingStrings.add(context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(marker.name)) + for(route in routes) + biasingStrings.add(context.getString(R.string.voice_cmd_start_route).format(route.name)) + + putStringArrayListExtra(RecognizerIntent.EXTRA_BIASING_STRINGS, biasingStrings) + } + } + + fun destroy() { + destroyRecognizer() + } + + private fun destroyRecognizer() { + speechRecognizer?.destroy() + speechRecognizer = null + } + + private val listener = object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + _state.value = VoiceCommandState.Listening + } + + override fun onResults(results: Bundle?) { + _state.value = VoiceCommandState.Idle + val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + matches?.forEach { println("speech: $it") } + if (matches != null) + handleSpeech(matches) + } + + override fun onError(error: Int) { + println("onError $error") + destroyRecognizer() + _state.value = VoiceCommandState.Error + onError() + } + + override fun onEndOfSpeech() {} + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onPartialResults(partialResults: Bundle?) {} + override fun onEvent(eventType: Int, params: Bundle?) {} + } + + data class VoiceCommand(val stringId: Int, val action: (arg: ArrayList) -> Unit) + + val simpleCommands = arrayOf( + VoiceCommand(R.string.directions_my_location) { service.myLocation() }, + VoiceCommand(R.string.help_orient_page_title) { service.whatsAroundMe() }, + VoiceCommand(R.string.help_explore_page_title) { service.aheadOfMe() }, + VoiceCommand(R.string.callouts_nearby_markers) { service.nearbyMarkers() }, + VoiceCommand(R.string.route_detail_action_next) { service.routeSkipNext() }, + VoiceCommand(R.string.route_detail_action_previous) { service.routeSkipPrevious() }, + VoiceCommand(R.string.beacon_action_mute_beacon) { service.routeMute() }, + VoiceCommand(R.string.route_detail_action_stop_route) { service.routeStop() }, + VoiceCommand(R.string.voice_cmd_list_routes) { service.routeListRoutes() }, + VoiceCommand(R.string.voice_cmd_list_markers) { service.routeListMarkers() }, + VoiceCommand(R.string.menu_help) { voiceHelp() }, + ) + + fun matchDynamicMarkers(speech: String): Boolean{ + val markers = listOfMarkers + val routes = listOfRoutes + + var minMatch = Double.MAX_VALUE + + // Markers + var bestMarker : MarkerEntity? = null + for(marker in markers) { + val commandString = context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(marker.name).lowercase() + println("Marker compare \"$commandString\" with \"$speech\"") + val match = commandString.fuzzyCompare(speech, false) + if ((match < 0.2) && (match < minMatch)) { + minMatch = match + bestMarker = marker + } + } + if(bestMarker != null) { + service.markerStart(bestMarker) + return true + } + + // Routes + var bestRoute : RouteEntity? = null + for(route in routes) { + val commandString = context.getString(R.string.voice_cmd_start_route).format(route.name).lowercase() + println("Route compare \"$commandString\" with \"$speech\"") + val match = commandString.fuzzyCompare(speech, false) + if ((match < 0.2) && (match < minMatch)) { + minMatch = match + bestRoute = route + } + } + if(bestRoute != null) { + service.routeStart(bestRoute) + return true + } + + return false + } + + private fun handleSpeech(speech: ArrayList) { + var bestMatch: VoiceCommand? = null + for(text in speech) { + + // Find the best match to the speech. + val t = text.lowercase() + + var minMatch = Double.MAX_VALUE + // Start with simpleCommands which don't contain any dynamic arguments + for (command in simpleCommands) { + val commandString = context.getString(command.stringId).lowercase() + val match = commandString.fuzzyCompare(t, false) + if ((match < 0.3) && (match < minMatch)) { + minMatch = match + bestMatch = command + } + } + if(bestMatch != null) + break + println("No simple command match found") + + // Check if we match any of our dynamic markers + if(matchDynamicMarkers(t)) + return + } + if (bestMatch != null) { + println("Found command: ${context.getString(bestMatch.stringId)}") + + // Pass in all the speech strings, it may be that the argument is clearer in ones other + // than our best match. + bestMatch.action(speech) + } else { + service.speak2dText( + context.getString(R.string.voice_cmd_not_recognized).format(speech.first()), + false, + EARCON_CALLOUTS_OFF + ) + } + } + + private fun voiceHelp() { + + val firstRoute = listOfRoutes.firstOrNull()?.name + val firstMarker = listOfMarkers.firstOrNull()?.name + + val commandNames = simpleCommands.map { context.getString(it.stringId) } + val builder = StringBuilder() + builder.append(context.getString(R.string.voice_cmd_help_response)) + commandNames.forEach { builder.append(it).append(". ") } + + if((firstRoute != null) || (firstMarker != null)) + builder.append(context.getString(R.string.voice_cmd_explain_dynamic_markers)) + if(firstRoute != null) + builder.append(context.getString(R.string.voice_cmd_start_route).format(firstRoute)).append(". ") + if(firstMarker != null) + builder.append(context.getString(R.string.voice_cmd_start_beacon_at_marker_with_name).format(firstMarker)).append(". ") + + service.speak2dText(builder.toString()) + } +} diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt index 4d21aa584..48f6d633d 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeState.kt @@ -18,4 +18,5 @@ data class HomeState( var routesTabSelected: Boolean = true, var currentRouteData: RoutePlayerState = RoutePlayerState(), var permissionsRequired: Boolean = false, + var voiceCommandListening: Boolean = false, ) diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt index bd3006c95..af550eeae 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/viewmodels/home/HomeViewModel.kt @@ -20,6 +20,7 @@ import org.scottishtecharmy.soundscape.SoundscapeServiceConnection import org.scottishtecharmy.soundscape.audio.AudioTour import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription +import org.scottishtecharmy.soundscape.services.mediacontrol.VoiceCommandState import javax.inject.Inject @HiltViewModel @@ -106,6 +107,12 @@ class HomeViewModel _state.update { it.copy(currentRouteData = value) } } } + viewModelScope.launch(job!!) { + // Observe voice command listening state + soundscapeServiceConnection.getVoiceCommandStateFlow()?.collectLatest { voiceState -> + _state.update { it.copy(voiceCommandListening = voiceState is VoiceCommandState.Listening) } + } + } } private fun stopMonitoringLocation() { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 6b695714d..cfb9aa798 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -478,6 +478,13 @@ "Ahead\nof Me" "Hear about places in front of you" + +"Route control" +"Next waypoint" +"Previous waypoint" +"No routes saved" +"Top menu" + "Menu" @@ -829,6 +836,10 @@ No language selected Notifications + +Record audio + +This permission is optional, but is required if you want to use voice commands to control the app. The Soundscape location service runs even when the phone is locked. @@ -839,6 +850,18 @@ "Language and Region " "Accessibility" + +"Media Controls" + +"Media controls mode" + +"The media control buttons on your headphones, or remote can be configured to work in different ways depending on this option." + +"Original mode" + +"Voice command" + +"Audio menu" "Light or Dark Theme" @@ -2576,4 +2599,38 @@ Please report any problems that you have, however small they may be via the *Con Continue Close the popup and continue with the tutorial + +Available voice commands: + +Listening… + +"I'm sorry, I can't find the command %s" + +List routes + +Available routes are, + +List markers + +Available markers are, + +No routes have been saved + +No markers have been saved + +Route not found: %s + +Starting route %s + +Start route %s + + + +Start audio beacon at %s + +Starting audio beacon at %s + +Marker not found: %s + +You can also start routes and markers by name, for example. diff --git a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt index 4201b841a..ea81879fb 100644 --- a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt +++ b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt @@ -269,7 +269,7 @@ class MvtTileTest { TreeId.SETTLEMENT_VILLAGE, TreeId.SETTLEMENT_HAMLET, TreeId.TRANSIT -> - {} + {} else -> { for (feature in collection) { @@ -591,7 +591,7 @@ class MvtTileTest { val w1 = 300.0 val w2 = 100.0 val fitness = (w1 * (10 / (10 + sensedRoadInfo.distance))) + - (w2 * (30 / (30 + headingOffSensedRoad))) + (w2 * (30 / (30 + headingOffSensedRoad))) if(fitness > bestFitness) { bestFitness = fitness bestIndex = index @@ -968,18 +968,23 @@ class MvtTileTest { LngLatAlt(location.longitude, location.latitude), emptySet() ) + if(false) { + // This code is useful for comparing before and after changes of grid parsing + val adapter = GeoJsonObjectMoshiAdapter() + val tileOutput = FileOutputStream("cache-test2/output-$x-$y.geojson") + + // Output the GeoJson and check that there's no data left from other tiles. + val collection = FeatureCollection() + for (id in TreeId.entries) { + if (id < TreeId.MAX_COLLECTION_ID) + collection += gridState.getFeatureCollection(id) + } + tileOutput.write(adapter.toJson(collection).toByteArray()) + tileOutput.close() + } } } } -// val adapter = GeoJsonObjectMoshiAdapter() -// val mapMatchingOutput = FileOutputStream("total-output.geojson") -// -// // Output the GeoJson and check that there's no data left from other tiles. -// val collection = gridState.getFeatureCollection(TreeId.WAYS_SELECTION) -// collection += gridState.getFeatureCollection(TreeId.INTERSECTIONS) -// collection += gridState.getFeatureCollection(TreeId.POIS) -// mapMatchingOutput.write(adapter.toJson(collection).toByteArray()) -// mapMatchingOutput.close() } @Test diff --git a/docs/developers/voice-and-audio-control.md b/docs/developers/voice-and-audio-control.md new file mode 100644 index 000000000..80e5f4603 --- /dev/null +++ b/docs/developers/voice-and-audio-control.md @@ -0,0 +1,124 @@ +--- +title: App remote control +layout: page +parent: Information for developers +has_toc: false +--- +# App remote control +As well as the accessible GUI to control the app, we also have 3 other options which can control the app even when the screen is locked: + +1. Hard coded media controls as per the iOS app +2. Voice control +3. An audio menu navigated via media controls + +The first of these was implemented in an early release, but the second two are new. + +## Voice control +Most voice commands will be relatively fixed and discoverable simply by asking. The expected design is that the user will tap the play/pause button on their headphones and that will prompt the user that it's listening for a command. The user speaks the command and after a short period of silence the listening will stop and the text is sent to the app to process. + +It's possible to have a number of phrases that have the same action, though it's not clear whether that really makes sense. If you can use "My location" to trigger that feature, then having "Where am I" as well might just be confusing? In general sticking to the same text as is used in the GUI makes the most sense. To do that, we should actually use the same string resources so that the translations remain consistent. + +Because we want the functionality of voice control and audio menus to be similar, I've added in the text alongside the audio menu options in the next section. + +## Audio menus +Each menu consists of a number of options to pick from. The main menu is simply a list of these sub menus. Each sub menu has an option which allows navigating back up to the main menu. That option obviously doesn't exist as a voice control command where all commands are in a flat structure. + +### Existing features +#### My Surroundings +Selecting an option simply triggers in the same way as tapping the UI button. +``` +My Surroundings +├── "My Location" +├── "Around Me" +├── "Ahead of Me" +├── "Nearby Markers" +└── Main Menu +``` + +#### Route control +These are the actions that are hard coded in iOS when a route is playing back. All button presses must have obvious audio feedback even if it's just to say "You're already at the first waypoint". +``` +Route +├── "Next Waypoint" +├── "Prev Waypoint" +├── "Mute Beacon" +├── "Stop Route" +└── Main Menu +``` + +#### Start route +This is the first of our dynamically generated menus. It contains every route that has been set up. The routes should be sorted alphabetically. +``` +Start Route +├── [route 1] +├── [route 2] +├── ... +├── ... +└── Main Menu +``` + +Two voice commands cover this: +1. "List routes" +2. "Start route [route name]" + +This means that the user can discover the available routes and then start playing one back. + +#### Start beacon +This is the same as start route, but for Markers. Many users don't create routes and so have fewer markers to scroll through. The markers should be sorted alphabetically. +``` +Start Beacon +├── [marker 1] +├── [marker 2] +├── ... +├── ... +└── Main Menu +``` + +Again, two voice commands cover this: +1. "List markers" +2. "Start beacon at [marker name]" + +Perhaps we could add filters to "List markers" e.g. "List markers nearby" or "List markers from F" where it start alphabetically. + +### New features +One of the main reasons for having an extensible menu system is so that we can add in new remotely controlled features. We need to decide how we expose these within the on screen GUI too - in many ways it's a lot easier not to! + +#### Audio profile +This is a new feature which would allow the user to pick a filter for points of interest. I think the implementation of the filters is the crucial thing here. Rather than ONLY playing out eating POI, perhaps it guarantees eating POI, but will play some other POI to aid navigation e.g. at least one POI per 100m if there have been no intersections? +``` +Audio Profile +├── "Eating" +├── "Shopping" +├── "Banking" +├── "Transport" +├── "Navigating" +├── ... +├── ... +└── Main Menu +``` + +#### Route creation +The UI for creating routes is hard to use, and the idea here is to allow users to create routes as they walk. +``` +Route creation +├── "Start creating route" +├── "Add waypoint" +└── Main Menu +``` + +Voice control does have some major advantages as we can allow the user to specify names too. For example: + +* "Start creating route 'To Tesco'" +* "Add waypoint 'Corner of Moor Road'" + +If the name is empty, we can either guess a good marker name or fall back to using "Waypoint X". + +#### Callback filtering +This sort of goes along with audio profiles, but it simply a way of getting the app to quieten down when there's too much to describe. Perhaps this should just be an audio profile e.g. "Quieter"? +``` +Callback filtering +├── Mute callbacks +├── Reduce callbacks +├── Increase callbacks +└── Main Menu +``` diff --git a/docs/documentationScreens/homeScreen.png b/docs/documentationScreens/homeScreen.png index bde723af3..f623f36c0 100644 Binary files a/docs/documentationScreens/homeScreen.png and b/docs/documentationScreens/homeScreen.png differ diff --git a/docs/documentationScreens/homeScreenWithRoute.png b/docs/documentationScreens/homeScreenWithRoute.png index 679fd3044..449edc83d 100644 Binary files a/docs/documentationScreens/homeScreenWithRoute.png and b/docs/documentationScreens/homeScreenWithRoute.png differ diff --git a/docs/documentationScreens/routeDetails.png b/docs/documentationScreens/routeDetails.png index 6930c4647..406d7452f 100644 Binary files a/docs/documentationScreens/routeDetails.png and b/docs/documentationScreens/routeDetails.png differ diff --git a/docs/documentationScreens/routeEdit.png b/docs/documentationScreens/routeEdit.png index fda8b037b..8e0d4fdd9 100644 Binary files a/docs/documentationScreens/routeEdit.png and b/docs/documentationScreens/routeEdit.png differ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ad6d2b6a8..b74a2028b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,7 @@ protobuf-plugin = "0.9.6" dagger-hilt = "2.57" devtools-ksp = "2.3.2" google-services = "4.4.3" -jetbrains-dokka = "2.0.0" +jetbrains-dokka = "2.2.0-Beta" firebase-crashlytics = "3.0.6" accompanistPermissions = "0.37.3" androidGpxParser = "2.3.1"