diff --git a/app-release-signed.apk b/app-release-signed.apk index e6aa551..f14a151 100644 Binary files a/app-release-signed.apk and b/app-release-signed.apk differ diff --git a/app-release-signed.apk.idsig b/app-release-signed.apk.idsig index e39af99..58f1c35 100644 Binary files a/app-release-signed.apk.idsig and b/app-release-signed.apk.idsig differ diff --git a/app/src/main/kotlin/com/google/ai/sample/MenuScreen.kt b/app/src/main/kotlin/com/google/ai/sample/MenuScreen.kt index 41d67d5..d0963bd 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MenuScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MenuScreen.kt @@ -274,7 +274,14 @@ fun MenuScreen( .padding(horizontal = 16.dp, vertical = 8.dp) ) { val annotatedText = buildAnnotatedString { - append("Preview models could be deactivated by Google without being handed over to the final release. Gemma 3n E4B it cannot handle screenshots in the API. There are rate limits for free use of Gemini models. The less powerful the models are, the more you can use them. The limits range from a maximum of 5 to 30 calls per minute. After each screenshot (every 2-3 seconds) the LLM must respond again. More information is available at ") + append("• Preview models could be deactivated by Google without being handed over to the final release.\\n") + append("• GPT-oss 120b is a pure text model.\\n") + append("• Gemma 3n E4B it cannot handle screenshots in the API.\\n") + append("• GPT models (Vercel) have a free budget of $5 per month.\\n") + append("GPT-5.1 Input: $1.25/M Output: $10.00/M\\n") + append("GPT-5.1 mini Input: $0.25/ M Output: $2.00/M\\n") + append("GPT-5 nano Input: $0.05/M Output: $0.40/M\\n") + append("• There are rate limits for free use of Gemini models. The less powerful the models are, the more you can use them. The limits range from a maximum of 5 to 30 calls per minute. After each screenshot (every 2-3 seconds) the LLM must respond again. More information is available at ") pushStringAnnotation(tag = "URL", annotation = "https://ai.google.dev/gemini-api/docs/rate-limits") withStyle(style = SpanStyle(color = MaterialTheme.colorScheme.primary, textDecoration = TextDecoration.Underline)) { diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt index c2499a1..371f72c 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt @@ -40,11 +40,20 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.decodeFromString import kotlinx.serialization.MissingFieldException +import kotlinx.serialization.json.JsonClassDiscriminator +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.modules.polymorphic import androidx.core.app.NotificationCompat import androidx.localbroadcastmanager.content.LocalBroadcastManager +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import java.io.File import java.io.FileOutputStream import java.text.SimpleDateFormat @@ -70,6 +79,7 @@ class ScreenCaptureService : Service() { const val EXTRA_AI_CHAT_HISTORY_JSON = "com.google.ai.sample.EXTRA_AI_CHAT_HISTORY_JSON" const val EXTRA_AI_MODEL_NAME = "com.google.ai.sample.EXTRA_AI_MODEL_NAME" // For service to create model const val EXTRA_AI_API_KEY = "com.google.ai.sample.EXTRA_AI_API_KEY" // For service to create model + const val EXTRA_AI_API_PROVIDER = "com.google.ai.sample.EXTRA_AI_API_PROVIDER" // For service to select API const val EXTRA_TEMP_FILE_PATHS = "com.google.ai.sample.EXTRA_TEMP_FILE_PATHS" @@ -189,6 +199,8 @@ class ScreenCaptureService : Service() { val chatHistoryJson = intent.getStringExtra(EXTRA_AI_CHAT_HISTORY_JSON) val modelName = intent.getStringExtra(EXTRA_AI_MODEL_NAME) val apiKey = intent.getStringExtra(EXTRA_AI_API_KEY) + val apiProviderString = intent.getStringExtra(EXTRA_AI_API_PROVIDER) + val apiProvider = ApiProvider.valueOf(apiProviderString ?: ApiProvider.GOOGLE.name) val tempFilePaths = intent.getStringArrayListExtra(EXTRA_TEMP_FILE_PATHS) ?: ArrayList() Log.d(TAG, "Received tempFilePaths for cleanup: $tempFilePaths") @@ -253,22 +265,28 @@ class ScreenCaptureService : Service() { } } try { - val generativeModel = GenerativeModel( - modelName = modelName, - apiKey = apiKey - ) - val tempChat = generativeModel.startChat(history = chatHistory) - val fullResponse = StringBuilder() - tempChat.sendMessageStream(inputContent).collect { chunk -> - chunk.text?.let { - fullResponse.append(it) - val streamIntent = Intent(ACTION_AI_STREAM_UPDATE).apply { - putExtra(EXTRA_AI_STREAM_CHUNK, it) + if (apiProvider == ApiProvider.VERCEL) { + val result = callVercelApi(modelName, apiKey, chatHistory, inputContent) + responseText = result.first + errorMessage = result.second + } else { + val generativeModel = GenerativeModel( + modelName = modelName, + apiKey = apiKey + ) + val tempChat = generativeModel.startChat(history = chatHistory) + val fullResponse = StringBuilder() + tempChat.sendMessageStream(inputContent).collect { chunk -> + chunk.text?.let { + fullResponse.append(it) + val streamIntent = Intent(ACTION_AI_STREAM_UPDATE).apply { + putExtra(EXTRA_AI_STREAM_CHUNK, it) + } + LocalBroadcastManager.getInstance(applicationContext).sendBroadcast(streamIntent) } - LocalBroadcastManager.getInstance(applicationContext).sendBroadcast(streamIntent) } + responseText = fullResponse.toString() } - responseText = fullResponse.toString() } catch (e: MissingFieldException) { Log.e(TAG, "Serialization error, potentially a 503 error.", e) // Check if the error message indicates a 503-like error @@ -680,3 +698,116 @@ class ScreenCaptureService : Service() { override fun onBind(intent: Intent?): IBinder? = null } + +// Data classes for Vercel API +@Serializable +data class VercelRequest( + val model: String, + val messages: List +) + +@Serializable +data class VercelMessage( + val role: String, + val content: List +) + +@Serializable +data class VercelResponse( + val choices: List +) + +@Serializable +data class VercelChoice( + val message: VercelResponseMessage +) + +@Serializable +data class VercelResponseMessage( + val role: String, + val content: String +) + +@Serializable +@JsonClassDiscriminator("type") +sealed class VercelContent + +@Serializable +@SerialName("text") +data class VercelTextContent(val text: String) : VercelContent() + +@Serializable +@SerialName("image_url") +data class VercelImageContent(val image_url: VercelImageUrl) : VercelContent() + +@Serializable +data class VercelImageUrl(val url: String) + +private fun Bitmap.toBase64(): String { + val outputStream = java.io.ByteArrayOutputStream() + this.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) + return "data:image/jpeg;base64," + android.util.Base64.encodeToString(outputStream.toByteArray(), android.util.Base64.DEFAULT) +} + +private suspend fun callVercelApi(modelName: String, apiKey: String, chatHistory: List, inputContent: Content): Pair { + var responseText: String? = null + var errorMessage: String? = null + + val json = Json { + serializersModule = SerializersModule { + polymorphic(VercelContent::class) { + subclass(VercelTextContent::class, VercelTextContent.serializer()) + subclass(VercelImageContent::class, VercelImageContent.serializer()) + } + } + ignoreUnknownKeys = true + } + + try { + val messages = (chatHistory + inputContent).map { content -> + val parts = content.parts.map { part -> + when (part) { + is TextPart -> VercelTextContent(text = part.text) + is ImagePart -> VercelImageContent(image_url = VercelImageUrl(url = part.image.toBase64())) + else -> VercelTextContent(text = "") // Or handle other part types appropriately + } + } + VercelMessage(role = if (content.role == "user") "user" else "assistant", content = parts) + } + + val requestBody = VercelRequest( + model = modelName, + messages = messages + ) + + val client = OkHttpClient() + val mediaType = "application/json".toMediaType() + val jsonBody = json.encodeToString(VercelRequest.serializer(), requestBody) + + val request = Request.Builder() + .url("https://api.vercel.ai/v1/chat/completions") + .post(jsonBody.toRequestBody(mediaType)) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .build() + + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + errorMessage = "Unexpected code ${response.code} - ${response.body?.string()}" + } else { + val responseBody = response.body?.string() + if (responseBody != null) { + val json = Json { ignoreUnknownKeys = true } + val vercelResponse = json.decodeFromString(VercelResponse.serializer(), responseBody) + responseText = vercelResponse.choices.firstOrNull()?.message?.content ?: "No response from model" + } else { + errorMessage = "Empty response body" + } + } + } + } catch (e: Exception) { + errorMessage = e.localizedMessage ?: "Vercel API call failed" + } + + return Pair(responseText, errorMessage) +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index 50fbd1b..98b352e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -859,6 +859,7 @@ class PhotoReasoningViewModel( putExtra(ScreenCaptureService.EXTRA_AI_API_KEY, apiKey) // Add the new extra for file paths putStringArrayListExtra(ScreenCaptureService.EXTRA_TEMP_FILE_PATHS, tempFilePaths) + putExtra(ScreenCaptureService.EXTRA_AI_API_PROVIDER, currentModel.apiProvider.name) } context.startService(serviceIntent) Log.d(TAG, "sendMessageWithRetry: Sent intent to ScreenCaptureService to execute AI call.") diff --git a/build_and_sign.sh b/build_and_sign.sh old mode 100644 new mode 100755