Make the app available on Android 16, support gesture training together with Talkback

app/build.gradle

@@ -2,18 +2,18 @@ plugins {
     id 'com.android.application'
     id 'org.jetbrains.kotlin.android'
     id 'kotlin-android'
-    id 'kotlin-android-extensions'
+
     id 'com.google.gms.google-services'
     id 'com.google.firebase.crashlytics'
 }
 
 android {
-    compileSdk 33
+    compileSdk 36
 
     defaultConfig {
         applicationId 'app.screenreader'
         minSdk 24
-        targetSdk 33
+        targetSdk 36
         versionCode 9
         versionName '1.1.0'
     }
@@ -29,11 +29,11 @@ android {
         }
     }
     compileOptions {
-        sourceCompatibility JavaVersion.VERSION_1_8
-        targetCompatibility JavaVersion.VERSION_1_8
+        sourceCompatibility JavaVersion.VERSION_17
+        targetCompatibility JavaVersion.VERSION_17
     }
     kotlinOptions {
-        jvmTarget = '1.8'
+        jvmTarget = '17'
     }
     namespace 'app.screenreader'
 }

app/src/main/AndroidManifest.xml

@@ -31,5 +31,18 @@
         <activity
             android:name=".tabs.actions.ActionActivity"
             android:configChanges="orientation|screenSize" />
+
+        <service
+            android:name=".services.ScreenReaderService"
+            android:label="@string/service_label"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
+            android:exported="false">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/accessibility_service_config" />
+        </service>
     </application>
 </manifest>

app/src/main/java/app/screenreader/extensions/_Context.kt

@@ -1,8 +1,13 @@
 package app.screenreader.extensions
 
+import android.annotation.SuppressLint
 import android.app.Activity
+import android.content.BroadcastReceiver
 import android.content.Context
+import android.content.IntentFilter
 import android.net.Uri
+import android.os.Build
+import android.os.Build.VERSION.SDK_INT
 import android.os.LocaleList
 import android.text.SpannableString
 import android.text.Spanned
@@ -164,4 +169,23 @@ fun Context.openWebsite(uri: Uri) {
         .build()
 
     intent.launchUrl(this, uri)
+}
+
+@SuppressLint("UnspecifiedRegisterReceiverFlag")
+fun Context.registerBroadcastReceiver(
+    broadcastReceiver: BroadcastReceiver,
+    intentFilter: IntentFilter?,
+    exported: Boolean = true,
+) {
+
+    when {
+        SDK_INT >= Build.VERSION_CODES.TIRAMISU -> {
+            val exportedFlag =
+                if (exported) Context.RECEIVER_EXPORTED else Context.RECEIVER_NOT_EXPORTED
+
+            registerReceiver(broadcastReceiver, intentFilter, exportedFlag)
+        }
+
+        else -> registerReceiver(broadcastReceiver, intentFilter)
+    }
 }

app/src/main/java/app/screenreader/model/Constants.kt

@@ -11,5 +11,6 @@ class Constants {
         val SERVICE_ACTION = "SCREENREADER_SERVICE"
         val SERVICE_GESTURE = "GESTURE"
         val SERVICE_KILLED = "KILLED"
+        val SERVICE_MOTION_EVENT = "MOTION_EVENT"
     }
 }

app/src/main/java/app/screenreader/services/ScreenReaderService.kt

@@ -2,6 +2,7 @@ package app.screenreader.services
 
 import android.accessibilityservice.AccessibilityService
 import android.accessibilityservice.AccessibilityServiceInfo
+import android.accessibilityservice.TouchInteractionController
 import android.app.ActivityManager
 import android.content.Context
 import android.content.Intent
@@ -10,6 +11,7 @@ import android.os.Build
 import android.provider.Settings
 import android.util.Log
 import android.view.KeyEvent
+import android.view.MotionEvent
 import android.view.WindowManager
 import android.view.accessibility.AccessibilityEvent
 import android.view.accessibility.AccessibilityManager
@@ -18,6 +20,8 @@ import androidx.core.content.ContextCompat
 import androidx.core.hardware.display.DisplayManagerCompat
 import app.screenreader.MainActivity
 import app.screenreader.R
+import app.screenreader.tabs.actions.ActionActivity
+import app.screenreader.tabs.gestures.GestureActivity
 import app.screenreader.extensions.getSpannable
 import app.screenreader.model.Constants
 import app.screenreader.model.Gesture
@@ -34,16 +38,18 @@ import java.io.Serializable
 class ScreenReaderService: AccessibilityService() {
 
     private val TAG = "ScreenReaderService"
-    private val GESTURE_TRAINING_CLASS_NAME = MainActivity::class.java.name
+    private val MAIN_ACTIVITY_CLASS_NAME = MainActivity::class.java.name
+    private val GESTURE_ACTIVITY_CLASS_NAME = GestureActivity::class.java.name
+    private val ACTION_ACTIVITY_CLASS_NAME = ActionActivity::class.java.name
+
+    private var touchController: TouchInteractionController? = null
+    private var touchControllerCallback: TouchInteractionController.Callback? = null
 
     override fun onCreate() {
         super.onCreate()
         Log.i(TAG, "onCreate")
 
-        // Set passthrough regions
         setPassthroughRegions()
-
-        // Start GestureActivity
         startGestureTraining()
     }
 
@@ -71,10 +77,95 @@ class ScreenReaderService: AccessibilityService() {
     override fun onServiceConnected() {
         Log.i(TAG, "Service connected")
         super.onServiceConnected()
+
+        // Setup TouchInteractionController (API 32+) to intercept touch events
+        // and pass them through to the app via requestDelegating(), bypassing TalkBack's gesture handling
+        setupTouchInteractionController()
+    }
+
+    private fun setupTouchInteractionController() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S_V2) {
+            try {
+                // Use default display (ID = 0)
+                val displayId = android.view.Display.DEFAULT_DISPLAY
+
+                touchController = getTouchInteractionController(displayId)
+                Log.i(TAG, "Got TouchInteractionController for display $displayId")
+
+                touchControllerCallback = object : TouchInteractionController.Callback {
+                    override fun onMotionEvent(event: MotionEvent) {
+                        Log.d(TAG, "TouchController onMotionEvent: action=${event.action}, pointerCount=${event.pointerCount}")
+
+                        // When gesture training is active, request delegating to pass ALL events
+                        // through to the app without TalkBack processing them.
+                        // This allows the app's gesture recognizers to handle both swipes and taps.
+                        if (isGestureTraining()) {
+                            touchController?.let { controller ->
+                                if (controller.state == TouchInteractionController.STATE_TOUCH_INTERACTING) {
+                                    Log.d(TAG, "Requesting delegating mode to bypass TalkBack")
+                                    controller.requestDelegating()
+                                }
+                            }
+                        }
+                    }
+
+                    override fun onStateChanged(state: Int) {
+                        val stateName = when (state) {
+                            TouchInteractionController.STATE_CLEAR -> "CLEAR"
+                            TouchInteractionController.STATE_TOUCH_INTERACTING -> "TOUCH_INTERACTING"
+                            TouchInteractionController.STATE_TOUCH_EXPLORING -> "TOUCH_EXPLORING"
+                            TouchInteractionController.STATE_DRAGGING -> "DRAGGING"
+                            TouchInteractionController.STATE_DELEGATING -> "DELEGATING"
+                            else -> "UNKNOWN($state)"
+                        }
+                        Log.d(TAG, "TouchController state changed to: $stateName")
+                    }
+                }
+
+                touchController?.registerCallback(mainExecutor, touchControllerCallback!!)
+                Log.i(TAG, "Registered TouchInteractionController callback")
+
+            } catch (e: Exception) {
+                Log.e(TAG, "Failed to setup TouchInteractionController", e)
+            }
+        }
+    }
+
+    /**
+     * Called when raw motion events are received from the configured motion event sources.
+     * On API 32+, we use TouchInteractionController with requestDelegating() instead,
+     * which passes events directly to the app's normal touch pipeline.
+     */
+    override fun onMotionEvent(event: MotionEvent) {
+        Log.d(TAG, "onMotionEvent: action=${event.action}, pointerCount=${event.pointerCount}, x=${event.x}, y=${event.y}")
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S_V2) {
+            if (isGestureTraining()) {
+                broadcastMotionEvent(event)
+            }
+        }
+    }
+
+    private fun broadcastMotionEvent(event: MotionEvent) {
+        val intent = Intent(Constants.SERVICE_ACTION)
+        intent.setPackage(packageName)
+        // MotionEvent must be copied because the original may be recycled
+        intent.putExtra(Constants.SERVICE_MOTION_EVENT, MotionEvent.obtain(event))
+        sendBroadcast(intent)
     }
 
     override fun onUnbind(intent: Intent?): Boolean {
         Log.i(TAG, "onUnbind")
+
+        // Cleanup TouchInteractionController
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S_V2) {
+            touchControllerCallback?.let { callback ->
+                touchController?.unregisterCallback(callback)
+            }
+            touchController = null
+            touchControllerCallback = null
+        }
+
         return super.onUnbind(intent)
     }
 
@@ -85,34 +176,28 @@ class ScreenReaderService: AccessibilityService() {
     override fun onAccessibilityEvent(event: AccessibilityEvent?) {
         Log.i(TAG, "onAccessibilityEvent: $event")
 
-        // Continue if eventType = TYPE_WINDOW_STATE_CHANGED
         if (event == null || event.eventType != AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
             return
         }
 
-        // Continue if packageName is empty
         if (event.packageName == null || event.packageName.isEmpty()) {
             return
         }
 
-        // Continue if event does not come from own package
+
         if (event.packageName == this.packageName) {
             return
         }
-
-        // Continue if event does not come from accessibility package
         if (event.packageName.contains("accessibility")) {
             return
         }
 
-        // Continue if text does not contain the service label
         val serviceName = getString(R.string.service_label)
         if (event.text.contains(serviceName)) {
             return
         }
 
-        // Continue if the gesture training is not active
-        if (isGestureTraining()) {
+        if (isInApp()) {
             return
         }
 
@@ -121,11 +206,16 @@ class ScreenReaderService: AccessibilityService() {
     }
 
     override fun onGesture(gestureId: Int): Boolean {
-        Log.i(TAG, "onGesture: $gestureId")
+        Log.i(TAG, "onGesture called with gestureId: $gestureId")
 
         // Broadcast gesture to GestureActivity
-        Gesture.from(gestureId)?.let { gesture ->
+        val gesture = Gesture.from(gestureId)
+        Log.i(TAG, "Mapped gestureId $gestureId to gesture: $gesture")
+
+        if (gesture != null) {
             broadcast(Constants.SERVICE_GESTURE, gesture)
+        } else {
+            Log.w(TAG, "Unknown gestureId: $gestureId - not mapped to any Gesture")
         }
 
         // Kill service if touch exploration is disabled
@@ -158,7 +248,6 @@ class ScreenReaderService: AccessibilityService() {
             val flags = service.capabilities
             val capability = AccessibilityServiceInfo.CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION
 
-            // Check if Touch Exploration capability is granted
             if (flags and capability == capability) {
                 count++
             }
@@ -171,7 +260,19 @@ class ScreenReaderService: AccessibilityService() {
     private fun isGestureTraining(): Boolean {
         val activityManager = getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager
         activityManager.getRunningTasks(1).firstOrNull()?.topActivity?.let { activity ->
-            return activity.className == GESTURE_TRAINING_CLASS_NAME
+            // Check if the user is in the GestureActivity (where gesture training happens)
+            return activity.className == GESTURE_ACTIVITY_CLASS_NAME
+        }
+        return false
+    }
+
+    private fun isInApp(): Boolean {
+        val activityManager = getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager
+        activityManager.getRunningTasks(1).firstOrNull()?.topActivity?.let { activity ->
+            // Check if user is in any of the app's activities
+            return activity.className == MAIN_ACTIVITY_CLASS_NAME ||
+                   activity.className == GESTURE_ACTIVITY_CLASS_NAME ||
+                   activity.className == ACTION_ACTIVITY_CLASS_NAME
         }
         return false
     }
@@ -181,15 +282,15 @@ class ScreenReaderService: AccessibilityService() {
         gestures.forEach { gesture ->
             gesture.completed(this, false)
         }
-
-//        val intent = Intent(this, GestureActivity::class.java)
-//        intent.setGestures(gestures)
-//        intent.setInstructions(instructions)
-//        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
-//        startActivity(intent)
     }
 
     companion object {
+        const val MIN_API_FOR_TALKBACK_COMPATIBILITY = Build.VERSION_CODES.TIRAMISU
+
+        fun supportsTalkBackCompatibility(): Boolean {
+            return Build.VERSION.SDK_INT >= MIN_API_FOR_TALKBACK_COMPATIBILITY
+        }
+
         fun isEnabled(context: Context): Boolean {
             (context.getSystemService(Context.ACCESSIBILITY_SERVICE) as AccessibilityManager).let { manager ->
                 val services = manager.getEnabledAccessibilityServiceList(AccessibilityServiceInfo.FEEDBACK_ALL_MASK)

app/src/main/java/app/screenreader/tabs/actions/ActionActivity.kt

@@ -3,16 +3,15 @@ package app.screenreader.tabs.actions
 import android.accessibilityservice.AccessibilityServiceInfo
 import android.content.Intent
 import android.text.SpannableString
+import android.widget.ScrollView
 import app.screenreader.R
 import app.screenreader.extensions.doGetAction
-import app.screenreader.extensions.identifier
 import app.screenreader.extensions.showDialog
 import app.screenreader.helpers.Accessibility
 import app.screenreader.helpers.Events
 import app.screenreader.model.Action
 import app.screenreader.views.actions.ActionViewCallback
 import app.screenreader.widgets.ToolbarActivity
-import kotlinx.android.synthetic.main.activity_action.*
 
 /**
  * Created by Jan Jaap de Groot on 16/11/2020
@@ -22,6 +21,7 @@ class ActionActivity: ToolbarActivity(), ActionViewCallback {
 
     private val startTime = System.currentTimeMillis()
 
+    private val scrollView get() = findViewById<ScrollView>(R.id.scrollView)
     private val action: Action by lazy {
         intent.doGetAction() ?: Action.SELECT
     }