Skip to content

Commit 2c8e2dd

Browse files
google-labs-jules[bot]google-labs-jules[bot]
committed
feat: Use LocalBroadcastManager for robust communication
This commit refactors the communication between `ScreenCaptureService` and `PhotoReasoningViewModel` to use `LocalBroadcastManager`. This addresses the issue of broadcasts being lost when the app is in the background, which was causing the UI to get stuck in a loading state. `LocalBroadcastManager` ensures reliable internal communication within the app. The `ScreenCaptureService` has also been made more robust with improved error handling and a retry mechanism for the AI call.
1 parent 5929e8b commit 2c8e2dd

File tree

4 files changed

+105
-98
lines changed

4 files changed

+105
-98
lines changed

‎app/src/main/kotlin/com/google/ai/sample/MainActivity.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,6 +1134,7 @@ class MainActivity : ComponentActivity() {
11341134
const val EXTRA_SCREEN_INFO = "com.google.ai.sample.EXTRA_SCREEN_INFO"
11351135
}
11361136

1137+
11371138
override fun onNewIntent(intent: Intent?) {
11381139
super.onNewIntent(intent)
11391140
if (intent?.action == NotificationUtil.ACTION_STOP_OPERATION) {

‎app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt‎

Lines changed: 29 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import kotlinx.coroutines.launch
4343
import kotlinx.serialization.json.Json
4444
import kotlinx.serialization.decodeFromString
4545
import androidx.core.app.NotificationCompat
46+
import androidx.localbroadcastmanager.content.LocalBroadcastManager
4647
import java.io.File
4748
import java.io.FileOutputStream
4849
import java.text.SimpleDateFormat
@@ -244,54 +245,49 @@ class ScreenCaptureService : Service() {
244245
}
245246
}
246247
}
247-
248-
// Create a GenerativeModel instance for this specific call.
249-
// This ensures the call uses the API key and model name provided by the ViewModel.
250-
// Consider a default GenerationConfig or make it configurable too if needed.
251-
val generativeModel = GenerativeModel(
252-
modelName = modelName,
253-
apiKey = apiKey
254-
// generationConfig = generationConfig { ... } // Optional: add default config
255-
)
256-
257-
// Start a new chat session with the provided history for this call.
258-
val tempChat = generativeModel.startChat(history = chatHistory) // Use the mapped SDK history
259-
Log.d(TAG, "Executing AI sendMessage with history size: ${chatHistory.size}")
260-
val aiResponse = tempChat.sendMessage(inputContent) // Use the mapped SDK inputContent
261-
262-
if (aiResponse != null) {
263-
Log.d(TAG, "Service received AI Response. Success: true")
264-
// The response doesn't have a 'parts' property directly
265-
// If you need to log the response content, use the text property
266-
val responseLength = aiResponse.text?.length ?: 0
267-
Log.d(TAG, "Response text length: $responseLength")
268-
} else {
269-
Log.d(TAG, "Service received null AI Response object.")
248+
try {
249+
val generativeModel = GenerativeModel(
250+
modelName = modelName,
251+
apiKey = apiKey
252+
)
253+
val tempChat = generativeModel.startChat(history = chatHistory)
254+
val aiResponse = tempChat.sendMessage(inputContent)
255+
responseText = aiResponse?.text
256+
} catch (e: Exception) {
257+
Log.e(TAG, "Direct error in AI call", e)
258+
errorMessage = e.localizedMessage ?: "AI call failed"
259+
// Einmaliger Retry nur bei Netzwerk-Fehlern
260+
if (e.message?.contains("network") == true || e.message?.contains("connection") == true) {
261+
Log.d(TAG, "Retrying AI call once due to network error")
262+
kotlinx.coroutines.delay(500) // Kurzer Delay für Echtzeit
263+
val generativeModel = GenerativeModel(
264+
modelName = modelName,
265+
apiKey = apiKey
266+
)
267+
val tempChat = generativeModel.startChat(history = chatHistory)
268+
val aiResponse = tempChat.sendMessage(inputContent)
269+
responseText = aiResponse?.text
270+
}
270271
}
271-
responseText = aiResponse?.text // This line should remain
272-
Log.d(TAG, "AI call successful. Response text available: ${responseText != null}")
273272

274273
} catch (e: Exception) {
275274
// Catching general exceptions from model/chat operations or serialization
276-
Log.e(TAG, "Error during AI call execution in service", e)
277-
errorMessage = e.localizedMessage ?: "Unknown error during AI call in service"
278-
// More specific error handling (like API key failure leading to trying another key via ApiKeyManager)
279-
// could be added here if this service becomes responsible for ApiKeyManager interactions.
280-
// For "minimal changes", we just report the error back.
275+
Log.e(TAG, "Outer error during AI call execution", e)
276+
errorMessage = e.localizedMessage ?: "Unknown error"
281277
}
282278
finally {
283279
// Broadcast the result (success or error) back to the ViewModel.
284280
val resultIntent = Intent(ACTION_AI_CALL_RESULT).apply {
285-
`package` = applicationContext.packageName // Ensure only our app receives it
286281
if (responseText != null) {
287282
putExtra(EXTRA_AI_RESPONSE_TEXT, responseText)
288283
}
289284
if (errorMessage != null) {
290285
putExtra(EXTRA_AI_ERROR_MESSAGE, errorMessage)
291286
}
292287
}
293-
applicationContext.sendBroadcast(resultIntent)
294-
Log.d(TAG, "Broadcast sent for AI_CALL_RESULT. Error: $errorMessage, Response: ${responseText != null}")
288+
// Verwende LocalBroadcastManager für sichere, interne Broadcasts
289+
LocalBroadcastManager.getInstance(applicationContext).sendBroadcast(resultIntent)
290+
Log.d(TAG, "Local broadcast sent for AI_CALL_RESULT. Error: $errorMessage, Response: ${responseText != null}")
295291

296292
// Comment: Clean up temporary image files passed from the ViewModel.
297293
if (tempFilePaths.isNotEmpty()) {

‎app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import androidx.compose.foundation.layout.Column
1515
import androidx.compose.foundation.layout.PaddingValues
1616
import androidx.compose.foundation.layout.Row
1717
import androidx.compose.foundation.layout.Spacer
18+
import androidx.compose.foundation.layout.imePadding
1819
import androidx.compose.foundation.layout.calculateEndPadding
1920
import androidx.compose.foundation.layout.calculateStartPadding
2021
import androidx.compose.foundation.layout.fillMaxWidth
@@ -317,7 +318,8 @@ fun PhotoReasoningScreen(
317318
end = innerPadding.calculateEndPadding(LocalLayoutDirection.current) + 16.dp,
318319
top = 10.dp, // Kleines bisschen nach unten gerückt (verhindert Kollision mit Status-Bar-Schrift)
319320
bottom = innerPadding.calculateBottomPadding() + 16.dp
320-
),
321+
)
322+
.imePadding(), // NEU: Verschiebt das Layout nach oben wenn die Tastatur erscheint
321323
) {
322324
Card(
323325
modifier = Modifier.fillMaxWidth().padding(bottom = 16.dp),

‎app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt‎

Lines changed: 72 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.google.ai.sample.feature.multimodal
22

3+
import android.app.ActivityManager
4+
import android.app.ActivityManager.RunningAppProcessInfo
35
import android.content.Context
46
import android.graphics.Bitmap
57
import android.content.BroadcastReceiver
@@ -43,6 +45,7 @@ import kotlinx.serialization.json.Json
4345
import kotlinx.coroutines.Job
4446
import kotlinx.coroutines.flow.StateFlow
4547
import kotlinx.coroutines.flow.asStateFlow
48+
import androidx.localbroadcastmanager.content.LocalBroadcastManager
4649
// Removed duplicate StateFlow import
4750
// Removed duplicate asStateFlow import
4851
// import kotlinx.coroutines.isActive // Removed as we will use job.isActive
@@ -193,30 +196,25 @@ performReasoning(
193196
}
194197

195198
init {
196-
// ... other init logic if any ...
199+
// ... other init logic
197200
val context = MainActivity.getInstance()?.applicationContext
198201
if (context != null) {
199202
val filter = IntentFilter(ScreenCaptureService.ACTION_AI_CALL_RESULT)
200-
// Comment: Specify RECEIVER_NOT_EXPORTED for Android 13 (API 33) and above
201-
// to comply with security requirements for programmatically registered receivers.
202-
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
203-
context.registerReceiver(aiResultReceiver, filter, Context.RECEIVER_NOT_EXPORTED)
204-
} else {
205-
context.registerReceiver(aiResultReceiver, filter)
206-
}
207-
Log.d(TAG, "AIResultReceiver registered.")
203+
LocalBroadcastManager.getInstance(context).registerReceiver(aiResultReceiver, filter)
204+
Log.d(TAG, "AIResultReceiver registered with LocalBroadcastManager.")
208205
} else {
209206
Log.e(TAG, "Failed to register AIResultReceiver: applicationContext is null at init.")
210-
// Consider if this state implies a critical failure for the ViewModel's operation.
211-
// For now, just logging.
212207
}
213208
}
214209

215210
override fun onCleared() {
216211
super.onCleared()
217-
MainActivity.getInstance()?.applicationContext?.unregisterReceiver(aiResultReceiver)
218-
Log.d(TAG, "AIResultReceiver unregistered.")
219-
// ... other onCleared logic ...
212+
val context = MainActivity.getInstance()?.applicationContext
213+
if (context != null) {
214+
LocalBroadcastManager.getInstance(context).unregisterReceiver(aiResultReceiver)
215+
Log.d(TAG, "AIResultReceiver unregistered with LocalBroadcastManager.")
216+
}
217+
// ... other onCleared logic
220218
}
221219

222220
private fun createChatWithSystemMessage(context: Context? = null): Chat {
@@ -612,68 +610,78 @@ performReasoning(
612610
/**
613611
* Process commands found in the AI response
614612
*/
615-
private fun processCommands(text: String) {
616-
commandProcessingJob?.cancel() // Cancel any previous command processing
617-
commandProcessingJob = PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) {
618-
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch // Check for cancellation
619-
try {
620-
// Parse commands from the text
621-
val commands = CommandParser.parseCommands(text)
613+
private fun processCommands(text: String) {
614+
commandProcessingJob?.cancel() // Cancel any previous command processing
615+
commandProcessingJob = PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) {
616+
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch // Check for cancellation
617+
try {
618+
// Parse commands from the text
619+
val commands = CommandParser.parseCommands(text)
622620

623-
if (commands.isNotEmpty()) {
624-
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch
625-
Log.d(TAG, "Found ${commands.size} commands in response")
621+
// Check if takeScreenshot command is present
622+
val hasTakeScreenshotCommand = commands.any { it is Command.TakeScreenshot }
626623

627-
// Update the detected commands
628-
val currentCommands = _detectedCommands.value.toMutableList()
629-
currentCommands.addAll(commands)
630-
_detectedCommands.value = currentCommands
624+
if (commands.isNotEmpty()) {
625+
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch
626+
Log.d(TAG, "Found ${commands.size} commands in response")
631627

632-
// Update status to show commands were detected
633-
val commandDescriptions = commands.joinToString("; ") { command ->
634-
command.toString()
635-
}
636-
_commandExecutionStatus.value = "Commands detected: $commandDescriptions"
628+
// Update the detected commands
629+
val currentCommands = _detectedCommands.value.toMutableList()
630+
currentCommands.addAll(commands)
631+
_detectedCommands.value = currentCommands
637632

638-
// Execute the commands
639-
for (command in commands) {
640-
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation before executing each command
641-
Log.d(TAG, "Command execution stopped before executing: $command")
633+
// Update status to show commands were detected
634+
val commandDescriptions = commands.joinToString("; ") { command ->
635+
command.toString()
636+
}
637+
_commandExecutionStatus.value = "Commands detected: $commandDescriptions"
638+
639+
// Execute the commands
640+
for (command in commands) {
641+
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation before executing each command
642+
Log.d(TAG, "Command execution stopped before executing: $command")
643+
_commandExecutionStatus.value = "Command execution stopped."
644+
break // Exit loop if cancelled
645+
}
646+
try {
647+
Log.d(TAG, "Executing command: $command")
648+
ScreenOperatorAccessibilityService.executeCommand(command)
649+
// Check immediately after execution attempt if a stop was requested
650+
if (stopExecutionFlag.get()) {
651+
Log.d(TAG, "Command execution stopped after attempting: $command")
642652
_commandExecutionStatus.value = "Command execution stopped."
643-
break // Exit loop if cancelled
644-
}
645-
try {
646-
Log.d(TAG, "Executing command: $command")
647-
ScreenOperatorAccessibilityService.executeCommand(command)
648-
// Check immediately after execution attempt if a stop was requested
649-
if (stopExecutionFlag.get()) {
650-
Log.d(TAG, "Command execution stopped after attempting: $command")
651-
_commandExecutionStatus.value = "Command execution stopped."
652-
break
653-
}
654-
} catch (e: Exception) {
655-
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) break // Exit loop if cancelled during error handling
656-
Log.e(TAG, "Error executing command: ${e.message}", e)
657-
_commandExecutionStatus.value = "Error during command execution: ${e.message}"
653+
break
658654
}
659-
}
660-
if (stopExecutionFlag.get()){
661-
_commandExecutionStatus.value = "Command processing loop was stopped."
655+
} catch (e: Exception) {
656+
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) break // Exit loop if cancelled during error handling
657+
Log.e(TAG, "Error executing command: ${e.message}", e)
658+
_commandExecutionStatus.value = "Error during command execution: ${e.message}"
662659
}
663660
}
664-
} catch (e: Exception) {
665-
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch
666-
Log.e(TAG, "Error processing commands: ${e.message}", e)
667-
_commandExecutionStatus.value = "Error during command processing: ${e.message}"
668-
} finally {
669-
if (stopExecutionFlag.get()){
670-
_commandExecutionStatus.value = "Command processing finished after stop request."
661+
if (stopExecutionFlag.get()){
662+
_commandExecutionStatus.value = "Command processing loop was stopped."
663+
}
664+
}
665+
666+
// Toast anzeigen wenn kein takeScreenshot Command gefunden wurde
667+
if (!hasTakeScreenshotCommand && !text.contains("takeScreenshot()", ignoreCase = true)) {
668+
val context = MainActivity.getInstance()
669+
if (context != null) {
670+
Toast.makeText(context, "The AI stopped Screen Operator", Toast.LENGTH_SHORT).show()
671671
}
672-
// Reset flag after processing is complete or stopped to allow future executions
673-
// No, don't reset here. Reset at the beginning of 'reason' or when stop is explicitly cleared.
672+
}
673+
674+
} catch (e: Exception) {
675+
if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch
676+
Log.e(TAG, "Error processing commands: ${e.message}", e)
677+
_commandExecutionStatus.value = "Error during command processing: ${e.message}"
678+
} finally {
679+
if (stopExecutionFlag.get()){
680+
_commandExecutionStatus.value = "Command processing finished after stop request."
674681
}
675682
}
676683
}
684+
}
677685

678686
/**
679687
* Save chat history to SharedPreferences

0 commit comments

Comments
 (0)