Pocket code AI Integration face training and recognition

.gitignore

@@ -38,6 +38,9 @@ bin/
 gen/
 jniLibs/
 
+# unzipped by :catroid:unpackFaceTestAssets from faces.zip, do not commit
+catroid/src/androidTest/assets/faces/
+
 # Local configuration file (sdk path, etc)
 local.properties
 

catroid/build.gradle

@@ -105,6 +105,37 @@ ext.copyGoogleServicesFile = { flavorName, flavorId ->
 def defaultVersionCode = 102
 def defaultVersionName = "1.4.1"
 
+def faceTestArchive = file('src/androidTest/assets/faces.zip')
+// Unzipped straight into the conventional androidTest assets directory: AGP
+// already scans 'src/androidTest/assets' as an asset source by default, so
+// anything landing here is picked up with no extra sourceSet wiring. A
+// FileCollection.builtBy() srcDir hookup for a $buildDir output was tried and
+// does not get honoured by the asset merge task, so that indirection is
+// avoided entirely.
+def unpackedFaceTestAssetsDir = file('src/androidTest/assets/faces')
+
+tasks.register('unpackFaceTestAssets', Sync) {
+    group = 'verification'
+    description =
+            'Unpacks face-recognition fixtures for instrumented tests.'
+
+    inputs.file(faceTestArchive)
+    outputs.dir(unpackedFaceTestAssetsDir)
+
+    doFirst {
+        if (!faceTestArchive.isFile()) {
+            throw new GradleException(
+                    "Required face test archive is missing: $faceTestArchive"
+            )
+        }
+    }
+    from {
+        zipTree(faceTestArchive)
+    }
+
+    into unpackedFaceTestAssetsDir
+}
+
 android {
     compileSdk = 35
 
@@ -353,6 +384,11 @@ dependencies {
     } else {
         implementation fileTree(dir: 'src/main/libs', include: ['*.aar'])
     }
+    // Kotlin DSL
+    //implementation("org.tensorflow:tensorflow-lite:2.16.1")
+    implementation 'org.tensorflow:tensorflow-lite:2.12.0'
+    implementation 'org.tensorflow:tensorflow-lite-gpu:2.4.0'
+    implementation 'org.tensorflow:tensorflow-lite-support:0.2.0'
     implementation 'com.nostra13.universalimageloader:universal-image-loader:1.9.5'
     implementation ('com.esotericsoftware:kryo:5.1.1')
     implementation 'id.zelory:compressor:2.1.1'
@@ -364,6 +400,7 @@ dependencies {
     // Kotlin
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
+    testImplementation 'androidx.test:runner:1.6.1'
     androidTestImplementation project(path: ':catroid')
     androidTestImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutines_version"
     androidTestImplementation('androidx.arch.core:core-testing:2.2.0') {
@@ -485,7 +522,10 @@ dependencies {
     testImplementation "io.mockk:mockk:${mockkVersion}"
 
     testImplementation 'org.hamcrest:hamcrest-library:1.3'
-
+    testImplementation 'androidx.test:core:1.5.0'
+    androidTestImplementation 'androidx.test:runner:1.5.2'
+    androidTestImplementation 'androidx.test:rules:1.5.0'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
     testImplementation 'org.robolectric:robolectric:4.16.1'
 
     testImplementation 'org.reflections:reflections:0.9.11'
@@ -502,7 +542,6 @@ dependencies {
 
     androidTestImplementation "io.mockk:mockk-android:${mockkVersion}"
 
-    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
     androidTestImplementation 'androidx.test.ext:junit-ktx:1.1.5'
 
     androidTestImplementation "androidx.test.espresso:espresso-core:$espressoVersion"
@@ -555,6 +594,10 @@ tasks.configureEach { packageTask ->
     if (packageTask.name.contains("package")) {
         packageTask.dependsOn 'copyAndroidNatives'
     }
+    if ((packageTask.name.startsWith('merge') && packageTask.name.endsWith('AndroidTestAssets'))
+            || (packageTask.name.startsWith('process') && packageTask.name.endsWith('AndroidTestJavaRes'))) {
+        packageTask.dependsOn 'unpackFaceTestAssets'
+    }
 }
 
 tasks.withType(KotlinCompile).configureEach {

catroid/src/androidTest/java/org/catrobat/catroid/FaceRecognizer/FaceRecognizerLifecycleTest.kt

@@ -0,0 +1,104 @@
+package org.catrobat.catroid.FaceRecognizer
+
+import android.content.Context
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import org.catrobat.catroid.FaceRecognizer.env.FileUtils
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class FaceRecognizerLifecycleTest {
+
+    private lateinit var context: Context
+    private lateinit var recognizer: Recognizer
+
+    @Before
+    fun setUp() {
+        context = InstrumentationRegistry.getInstrumentation().targetContext
+        FileUtils.init(context)
+        FileUtils.deleteAll()
+        Recognizer.release()
+        recognizer = Recognizer.getInstance(context)
+    }
+
+    @After
+    fun tearDown() {
+        FileUtils.deleteAll()
+        Recognizer.release()
+    }
+
+    @Test
+    fun labelCanBeAddedAndIsVisibleImmediately() {
+        val index = recognizer.addPerson("Person A")
+
+        assertEquals(0, index)
+        assertEquals(listOf("Person A"), recognizer.classNames)
+    }
+
+    @Test
+    fun labelIsTrimmedAndDuplicateIsNotAdded() {
+        assertEquals(0, recognizer.addPerson("  Person A  "))
+        assertEquals(0, recognizer.addPerson("Person A"))
+        assertEquals(listOf("Person A"), recognizer.classNames)
+    }
+
+    @Test
+    fun blankLabelIsRejected() {
+        try {
+            recognizer.addPerson("   ")
+            fail("A blank label must be rejected")
+        } catch (_: IllegalArgumentException) {
+            // Expected.
+        }
+        assertTrue(recognizer.classNames.isEmpty())
+    }
+
+    @Test
+    fun labelsSurviveRecognizerRestart() {
+        recognizer.addPerson("Person A")
+        recognizer.addPerson("Person B")
+
+        Recognizer.release()
+        recognizer = Recognizer.getInstance(context)
+
+        assertEquals(listOf("Person A", "Person B"), recognizer.classNames)
+    }
+
+    @Test
+    fun deletingOneLabelKeepsTheRemainingLabelsInOrder() {
+        recognizer.addPerson("Person A")
+        recognizer.addPerson("Person B")
+        recognizer.addPerson("Person C")
+
+        recognizer.deletePerson(1)
+
+        assertEquals(listOf("Person A", "Person C"), recognizer.classNames)
+    }
+
+    @Test
+    fun deletedLabelStaysDeletedAfterRestart() {
+        recognizer.addPerson("Person A")
+        recognizer.addPerson("Person B")
+        recognizer.deletePerson(0)
+
+        Recognizer.release()
+        recognizer = Recognizer.getInstance(context)
+
+        assertEquals(listOf("Person B"), recognizer.classNames)
+    }
+
+    @Test
+    fun deletingUnknownIndexDoesNotDamageDatabase() {
+        recognizer.addPerson("Person A")
+
+        recognizer.deletePerson(99)
+
+        assertEquals(listOf("Person A"), recognizer.classNames)
+    }
+}

catroid/src/androidTest/java/org/catrobat/catroid/FaceRecognizer/RecognizerTrainingOrderRegressionTest.kt

@@ -0,0 +1,114 @@
+package org.catrobat.catroid.FaceRecognizer
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import org.catrobat.catroid.FaceRecognizer.env.FileUtils
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * Regression test for the historical "only the last-trained face is remembered" bug.
+ *
+ * The order is deliberately interleaved:
+ * train A, train B, recognise A, train C, recognise B.
+ * This must not be rewritten as "train everybody, then test everybody", because the
+ * interleaving is the behaviour under test.
+ */
+@RunWith(AndroidJUnit4::class)
+class RecognizerTrainingOrderRegressionTest {
+
+    private lateinit var appContext: Context
+    private lateinit var testContext: Context
+    private lateinit var recognizer: Recognizer
+
+    @Before
+    fun setUp() {
+        appContext = InstrumentationRegistry.getInstrumentation().targetContext
+        testContext = InstrumentationRegistry.getInstrumentation().context
+
+        FileUtils.init(appContext)
+        FileUtils.deleteAll()
+        Recognizer.release()
+        recognizer = Recognizer.getInstance(appContext)
+    }
+
+    @After
+    fun tearDown() {
+        FileUtils.deleteAll()
+        Recognizer.release()
+    }
+
+    @Test
+    fun earlierPeopleRemainRecognisableAfterLaterPeopleAreTrained() {
+        trainPerson("Person A", "p01")
+        trainPerson("Person B", "p02")
+
+        assertRecognisedAs("p01_test.jpg", "Person A")
+
+        trainPerson("Person C", "p03")
+
+        assertRecognisedAs("p02_test.jpg", "Person B")
+        assertEquals(listOf("Person A", "Person B", "Person C"), recognizer.classNames)
+    }
+
+    private fun trainPerson(name: String, assetPrefix: String) {
+        val personIndex = recognizer.addPerson(name)
+        var storedCount = 0
+
+        for (photoNumber in 1..4) {
+            val fileName = "${assetPrefix}_train$photoNumber.jpg"
+            val bitmap = requiredBitmap(fileName)
+            try {
+                val embeddings = recognizer.embedFrame(bitmap)
+                assertFalse(
+                    "$fileName produced no face embedding; the fixture or pipeline is invalid",
+                    embeddings.isEmpty()
+                )
+                storedCount = recognizer.addEmbeddings(personIndex, embeddings)
+            } finally {
+                bitmap.recycle()
+            }
+        }
+
+        assertTrue("No embeddings were stored for $name", storedCount > 0)
+    }
+
+    private fun assertRecognisedAs(fileName: String, expectedName: String) {
+        val bitmap = requiredBitmap(fileName)
+        try {
+            val result = recognizer.recognize(bitmap, true)
+            assertNotNull("$fileName was returned as Unknown", result)
+            assertEquals(
+                "$fileName was confused after another person was trained",
+                expectedName,
+                result?.name
+            )
+        } finally {
+            bitmap.recycle()
+        }
+    }
+
+    private fun requiredBitmap(fileName: String): Bitmap {
+        val assetPath = "faces/$fileName"
+        val bitmap = try {
+            testContext.assets.open(assetPath).use { input ->
+                BitmapFactory.decodeStream(input)
+            }
+        } catch (error: Exception) {
+            throw AssertionError("Required test asset is missing: $assetPath", error)
+        }
+
+        return requireNotNull(bitmap) {
+            "Required test asset could not be decoded: $assetPath"
+        }
+    }
+}

catroid/src/androidTest/java/org/catrobat/catroid/uiespresso/content/brick/app/LoadProjectAIExtensionSettingsTest.kt

@@ -124,6 +124,7 @@ import org.catrobat.catroid.formulaeditor.Sensors.SPEECH_RECOGNITION_LANGUAGE
 import org.catrobat.catroid.formulaeditor.Sensors.TEXT_FROM_CAMERA
 import org.catrobat.catroid.ui.MainMenuActivity
 import org.catrobat.catroid.ui.settingsfragments.SettingsFragment.SETTINGS_SHOW_AI_FACE_DETECTION_SENSORS
+import org.catrobat.catroid.ui.settingsfragments.SettingsFragment.SETTINGS_SHOW_AI_FACE_NAME_DETECTION_SENSORS
 import org.catrobat.catroid.ui.settingsfragments.SettingsFragment.SETTINGS_SHOW_AI_POSE_DETECTION_SENSORS
 import org.catrobat.catroid.ui.settingsfragments.SettingsFragment.SETTINGS_SHOW_AI_SPEECH_RECOGNITION_SENSORS
 import org.catrobat.catroid.ui.settingsfragments.SettingsFragment.SETTINGS_SHOW_AI_SPEECH_SYNTHETIZATION_SENSORS
@@ -162,12 +163,13 @@ class LoadProjectAIExtensionSettingsTest(
             *speechRecognitionList,
             *speechSynthetizationList,
             *faceDetectionList,
+            *facenamedetectionList,
             *poseDetectionList,
             *textRecognitionList
         )
 
         private var speechRecognitionLanguage = FormulaElement(SENSOR, SPEECH_RECOGNITION_LANGUAGE.name, null)
-
+        private var on_Device_Face_Recognition = FormulaElement(SENSOR, Sensors.ON_DEVICE_FACE_RECOGNITION.name, null)
         private var faceDetected = FormulaElement(SENSOR, FACE_DETECTED.name, null)
         private var faceSize = FormulaElement(SENSOR, FACE_SIZE.name, null)
         private var faceXPosition = FormulaElement(SENSOR, FACE_X.name, null)
@@ -285,7 +287,12 @@ class LoadProjectAIExtensionSettingsTest(
                 SpeakAndWaitBrick()
             )
         )
-
+        private val facenamedetectionList = arrayOf(
+            arrayOf(
+                "Face Detection faceDetected", SETTINGS_SHOW_AI_FACE_DETECTION_SENSORS,
+                ChangeSizeByNBrick(Formula(on_Device_Face_Recognition))
+            )
+        )
         private val faceDetectionList = arrayOf(
             arrayOf(
                 "Face Detection faceDetected", SETTINGS_SHOW_AI_FACE_DETECTION_SENSORS,
@@ -698,6 +705,7 @@ class LoadProjectAIExtensionSettingsTest(
         SETTINGS_SHOW_AI_SPEECH_RECOGNITION_SENSORS,
         SETTINGS_SHOW_AI_SPEECH_SYNTHETIZATION_SENSORS,
         SETTINGS_SHOW_AI_FACE_DETECTION_SENSORS,
+        SETTINGS_SHOW_AI_FACE_NAME_DETECTION_SENSORS,
         SETTINGS_SHOW_AI_POSE_DETECTION_SENSORS,
         SETTINGS_SHOW_AI_TEXT_RECOGNITION_SENSORS
     )