Bump

mobile/src/main/java/com/birdsounds/identify/MainActivity.kt

@@ -3,6 +3,7 @@ package com.birdsounds.identify
 import android.content.pm.PackageManager
 import android.os.Bundle
 import android.Manifest
+import android.annotation.SuppressLint
 import android.util.Log
 import androidx.activity.enableEdgeToEdge
 import androidx.appcompat.app.AppCompatActivity
@@ -13,7 +14,7 @@ import com.google.android.gms.wearable.ChannelClient
 import com.google.android.gms.wearable.Wearable
 
 class MainActivity : AppCompatActivity() {
-    private lateinit var soundClassifier: SoundClassifier
+//    private lateinit var soundClassifier: SoundClassifier
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -39,6 +40,8 @@ class MainActivity : AppCompatActivity() {
     }
     companion object {
         const val REQUEST_PERMISSIONS = 1337
+        @SuppressLint("StaticFieldLeak")
+        lateinit var soundClassifier: SoundClassifier
     }
 
     private fun requestPermissions() {

mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt

@@ -2,17 +2,48 @@ package com.birdsounds.identify
 
 
 import android.content.Intent
+import android.util.Half.abs
 import android.util.Log
 import androidx.localbroadcastmanager.content.LocalBroadcastManager
 import com.google.android.gms.wearable.MessageEvent
 import com.google.android.gms.wearable.WearableListenerService
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.ShortBuffer
 
 class MessageListenerService : WearableListenerService() {
     private val tag = "MessageListenerService"
+
+    //    fun placeSoundClassifier(soundClassifier: SoundClassifier)
     override fun onMessageReceived(p0: MessageEvent) {
         super.onMessageReceived(p0)
 
-//        Log.i(tag ,p0.data.toString(Charsets.UTF_8))
+        val soundclassifier = MainActivity.soundClassifier
+        if (soundclassifier == null) {
+            Log.w(tag, "Have invalid sound classifier")
+        } else {
+            Log.w(tag, "Have valid classifier")
+        }
+        val short_array = ShortArray(48000 * 3);
+        var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
+        var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
+
+
+        ByteBuffer.wrap(audio_bytes).order(
+            ByteOrder.LITTLE_ENDIAN
+        ).asShortBuffer().get(short_array)
+        Log.w(tag, short_array.sum().toString())
+        var sorted_list = soundclassifier.executeScoring(short_array)
+        Log.w(tag, "")
+        for (i in 0 until 5) {
+            val score = sorted_list[i].value
+            val index = sorted_list[i].index
+            val species_name = soundclassifier.labelList[index]
+            Log.w(tag, species_name + ", " + score.toString());
+        }
+        MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this)
+//        Log.i(tag , short_array.map( { abs(it)}).sum().toString())
+//        Log.i(tag, short_array[0].toString())
 //        Log.i(tag, p0.data.toString(Charsets.US_ASCII))
 //        broadcastMessage(p0)
     }

mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt

@@ -0,0 +1,57 @@
+package com.birdsounds.identify
+
+import android.content.Context
+import android.util.Log
+import com.google.android.gms.tasks.Tasks
+import com.google.android.gms.wearable.Wearable
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.launch
+import java.util.concurrent.ExecutionException
+
+object MessageSenderFromPhone {
+    const val tag = "MessageSender"
+    private val job = Job()
+    private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
+
+    fun sendMessage(path: String, message: ByteArray, context: Context) {
+        coroutineScope.launch {
+            sendMessageInBackground(path, message, context)
+        }
+    }
+
+    private fun sendMessageInBackground(path: String, message: ByteArray, context: Context) {
+        //first get all the nodes, ie connected wearable devices.
+        val nodeListTask = Wearable.getNodeClient(context).connectedNodes
+        try {
+            // Block on a task and get the result synchronously (because this is on a background
+            // thread).
+            val nodes = Tasks.await(nodeListTask)
+            if(nodes.isEmpty()) {
+                Log.i(tag,"No Node found to send message")
+            }
+            //Now send the message to each device.
+            for (node in nodes) {
+                val sendMessageTask = Wearable.getMessageClient(context)
+                    .sendMessage(node.id, path, message)
+                try {
+                    // Block on a task and get the result synchronously (because this is on a background
+                    // thread).
+                    val result = Tasks.await(sendMessageTask)
+                    Log.v(tag, "SendThread: message send to " + node.displayName)
+                } catch (exception: ExecutionException) {
+                    Log.e(tag, "Task failed: $exception")
+                } catch (exception: InterruptedException) {
+                    Log.e(tag, "Interrupt occurred: $exception")
+                }
+            }
+        } catch (exception: ExecutionException) {
+            Log.e(tag, "Task failed: $exception")
+        } catch (exception: InterruptedException) {
+            Log.e(
+                tag, "Interrupt occurred: $exception"
+            )
+        }
+    }
+}

mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt

@@ -22,6 +22,7 @@ import kotlin.concurrent.scheduleAtFixedRate
 import kotlin.math.ceil
 import kotlin.math.cos
 import uk.me.berndporr.iirj.Butterworth
+import java.nio.ShortBuffer
 import kotlin.math.round
 import kotlin.math.sin
 
@@ -31,6 +32,7 @@ class SoundClassifier(
 ) {
     internal var mContext: Context
     val TAG = "Sound Classifier"
+
     init {
         this.mContext = context.applicationContext
     }
@@ -58,10 +60,10 @@ class SoundClassifier(
 
 
     /** Names of the model's output classes.  */
-    lateinit var labelList: List<String>
+    public lateinit var labelList: List<String>
 
     /** Names of the model's output classes.  */
-    lateinit var assetList: List<String>
+    public lateinit var assetList: List<String>
 
     /** How many milliseconds between consecutive model inference calls.  */
     private var inferenceInterval = 800L
@@ -155,6 +157,7 @@ class SoundClassifier(
             }
             labelList = wordList.map { it.toTitleCase() }
             Log.i(TAG, "Label list entries: ${labelList.size}")
+
         } catch (e: IOException) {
             Log.e(TAG, "Failed to read labels ${filename}: ${e.message}")
         }
@@ -163,12 +166,16 @@ class SoundClassifier(
 
     private fun setupInterpreter(context: Context) {
         try {
-            val modelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.modelPath
+            val modelFilePath = context.getDir(
+                "filesdir",
+                Context.MODE_PRIVATE
+            ).absolutePath + "/" + options.modelPath
             Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
             val modelFile = File(modelFilePath)
-            val tfliteBuffer: ByteBuffer = FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
+            val tfliteBuffer: ByteBuffer =
-                channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
+                FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
-            }
+                    channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
+                }
             Log.i(TAG, "Done creating TFLite buffer from $modelFilePath")
 
             interpreter = Interpreter(tfliteBuffer, Interpreter.Options())
@@ -201,12 +208,16 @@ class SoundClassifier(
     private fun setupMetaInterpreter(context: Context) {
 
         try {
-            val metaModelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.metaModelPath
+            val metaModelFilePath = context.getDir(
+                "filesdir",
+                Context.MODE_PRIVATE
+            ).absolutePath + "/" + options.metaModelPath
             Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath")
             val metaModelFile = File(metaModelFilePath)
-            val tfliteBuffer: ByteBuffer = FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
+            val tfliteBuffer: ByteBuffer =
-                channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
+                FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
-            }
+                    channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
+                }
             Log.i(TAG, "Done creating TFLite buffer from $metaModelFilePath")
 
             meta_interpreter = Interpreter(tfliteBuffer, Interpreter.Options())
@@ -237,7 +248,7 @@ class SoundClassifier(
 
     fun runMetaInterpreter(location: Location) {
         val dayOfYear = LocalDate.now().dayOfYear
-        val week = ceil( dayOfYear*48.0/366.0) //model year has 48 weeks
+        val week = ceil(dayOfYear * 48.0 / 366.0) //model year has 48 weeks
         lat = location.latitude.toFloat()
         lon = location.longitude.toFloat()
 
@@ -280,6 +291,7 @@ class SoundClassifier(
 
         }
     }
+
     private fun generateDummyAudioInput(inputBuffer: FloatBuffer) {
         val twoPiTimesFreq = 2 * Math.PI.toFloat() * 1000f
         for (i in 0 until modelInputLength) {
@@ -287,6 +299,7 @@ class SoundClassifier(
             inputBuffer.put(i, sin(twoPiTimesFreq * x.toDouble()).toFloat())
         }
     }
+
     private fun String.toTitleCase() =
         splitToSequence("_")
             .map { it.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.ROOT) else it.toString() } }
@@ -297,17 +310,54 @@ class SoundClassifier(
         private const val TAG = "SoundClassifier"
         var lat: Float = 0.0f
         var lon: Float = 0.0f
+
         /** Number of nanoseconds in a millisecond  */
         private const val NANOS_IN_MILLIS = 1_000_000.toDouble()
     }
 
+    public fun executeScoring(
+        short_array: ShortArray
+    ): List<IndexedValue<Float>> {
+        val highPass = 0;
+        val butterworth = Butterworth()
+        butterworth.highPass(6, 48000.0, highPass.toDouble())
+
+        val outputBuffer = FloatBuffer.allocate(modelNumClasses)
+
+        for (i in 0 until modelInputLength) {
+            val s = short_array[i]
+
+            if (highPass == 0) inputBuffer.put(i, s.toFloat())
+            else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
+        }
+
+
+        inputBuffer.rewind()
+        outputBuffer.rewind()
+        interpreter.run(inputBuffer, outputBuffer)
+        outputBuffer.rewind()
+        outputBuffer.get(predictionProbs) // Copy data to predictionProbs.
+
+        val probList = mutableListOf<Float>()
+        for (i in predictionProbs.indices) {
+            probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i])))  //apply sigmoid
+        }
+        val outList = probList.withIndex().sortedByDescending { it -> it.value }
+//        Log.w(TAG, outList.toString())
+//        Log.i(TAG,labelList[outList[0].index].toString())
+        return outList
+
+
+
+    }
+
     private fun startRecognition() {
         if (modelInputLength <= 0 || modelNumClasses <= 0) {
             Log.e(TAG, "Switches: Cannot start recognition because model is unavailable.")
             return
         }
         val sharedPref = PreferenceManager.getDefaultSharedPreferences(mContext)
-        val highPass = sharedPref.getInt("high_pass",0)
+        val highPass = sharedPref.getInt("high_pass", 0)
         val butterworth = Butterworth()
         butterworth.highPass(6, 48000.0, highPass.toDouble())
 
@@ -315,7 +365,7 @@ class SoundClassifier(
 
         var j = 0 // Indices for the circular buffer next write
 
-        Log.w(TAG, "recognitionPeriod:"+inferenceInterval)
+        Log.w(TAG, "recognitionPeriod:" + inferenceInterval)
         recognitionTask = Timer().scheduleAtFixedRate(inferenceInterval, inferenceInterval) task@{
             val outputBuffer = FloatBuffer.allocate(modelNumClasses)
             val recordingBuffer = ShortArray(modelInputLength)
@@ -341,7 +391,7 @@ class SoundClassifier(
                 if (samplesAreAllZero && s.toInt() != 0) {
                     samplesAreAllZero = false
                 }
-                if (highPass==0)  inputBuffer.put(i, s.toFloat())
+                if (highPass == 0) inputBuffer.put(i, s.toFloat())
                 else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
             }
 
@@ -365,9 +415,9 @@ class SoundClassifier(
 //                    probList.add(1 / (1 + kotlin.math.exp(-value)))  //apply sigmoid
 //                }
 //            } else {
-                for (i in predictionProbs.indices) {
+            for (i in predictionProbs.indices) {
-                    probList.add( metaPredictionProbs[i] / (1+kotlin.math.exp(-predictionProbs[i])) )  //apply sigmoid
+                probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i])))  //apply sigmoid
-                }
+            }
 //            }
 
 //            if (mBinding.progressHorizontal.isIndeterminate){  //if start/stop button set to "running"
@@ -375,7 +425,7 @@ class SoundClassifier(
 //                    val max = it.maxByOrNull { entry -> entry.value }
 //                    updateTextView(max, mBinding.text1)
 //                    updateImage(max)
-                    //after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax)
+            //after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax)
 //                    val secondMax = it.filterNot { entry -> entry == max }.maxByOrNull { entry -> entry.value }
 //                    updateTextView(secondMax,mBinding.text2)
 //                }

wear/build.gradle.kts

@@ -67,6 +67,7 @@ dependencies {
     implementation(libs.androidx.core.splashscreen)
     implementation(libs.androidx.compose.navigation)
     implementation(libs.androidx.media3.common)
+    implementation("co.touchlab:stately-concurrent-collections:2.0.0")
     implementation(libs.androidx.compose.material3)
     implementation(libs.androidx.work.runtime.ktx) //    androidTestImplementation(platform(libs.androidx.compose.bom))
 //    androidTestImplementation(libs.androidx.ui.test.junit4)

wear/src/main/AndroidManifest.xml

@@ -34,6 +34,24 @@
                 <category android:name="android.intent.category.LAUNCHER" />
             </intent-filter>
         </activity>
+
+
+        <service
+            android:name=".MessageListenerService"
+            android:enabled="true"
+            android:exported="true" >
+            <intent-filter>
+                <action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
+
+                <data
+                    android:host="*"
+                    android:pathPrefix="/audio"
+                    android:scheme="wear" />
+            </intent-filter>
+        </service>
+
+
+
     </application>
 
 </manifest>

wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt

@@ -0,0 +1,35 @@
+package com.birdsounds.identify
+
+import android.app.Service
+
+
+
+import android.content.Intent
+import android.util.Half.abs
+import android.util.Log
+import androidx.localbroadcastmanager.content.LocalBroadcastManager
+import com.birdsounds.identify.presentation.MessageSender
+import com.google.android.gms.wearable.MessageEvent
+import com.google.android.gms.wearable.WearableListenerService
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.ShortBuffer
+
+class MessageListenerService : WearableListenerService() {
+    private val tag = "MessageListenerService"
+
+    //    fun placeSoundClassifier(soundClassifier: SoundClassifier)
+    override fun onMessageReceived(p0: MessageEvent) {
+        super.onMessageReceived(p0)
+
+
+        val t_scored = ByteBuffer.wrap(p0.data).getLong()
+        MessageSender.messageLog.add(t_scored)
+        Log.w("MSG_RECV", t_scored.toString())
+//        Log.i(tag , short_array.map( { abs(it)}).sum().toString())
+//        Log.i(tag, short_array[0].toString())
+//        Log.i(tag, p0.data.toString(Charsets.US_ASCII))
+//        broadcastMessage(p0)
+    }
+
+}

wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt

@@ -1,6 +1,7 @@
 package com.birdsounds.identify.presentation
 
 import android.Manifest
+import android.annotation.SuppressLint
 import android.app.Activity
 import android.content.Context
 import android.content.pm.PackageManager
@@ -12,6 +13,7 @@ import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.platform.LocalContext
 import androidx.core.content.ContextCompat
+import co.touchlab.stately.collections.ConcurrentMutableCollection
 import com.google.android.gms.tasks.Tasks
 import com.google.android.gms.wearable.ChannelClient
 import com.google.android.gms.wearable.ChannelClient.ChannelCallback
@@ -65,6 +67,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
         }
     }
 
+
     suspend fun permissionResultReturned() {
         playbackStateMutatorMutex.mutate {
             if (ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
@@ -136,6 +139,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
         //        // Stop recording
         //        recordingJob.cancel()
     }
+
 }
 
 object channelCallback : ChannelClient.ChannelCallback() {

wear/src/main/java/com/birdsounds/identify/presentation/MessageSendRecv.kt

@@ -1,11 +0,0 @@
-package com.birdsounds.identify.presentation
-
-import android.content.Context
-import androidx.work.CoroutineWorker
-import androidx.work.WorkerParameters
-import com.google.android.gms.wearable.Wearable
-
-class MessageSendRecv(private val context: Context)
-{
-//MainState.
-}

wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt

@@ -3,6 +3,9 @@ package com.birdsounds.identify.presentation
 
 import android.content.Context
 import android.util.Log
+import co.touchlab.stately.collections.ConcurrentMutableCollection
+import co.touchlab.stately.collections.ConcurrentMutableList
+import co.touchlab.stately.collections.ConcurrentMutableSet
 import com.google.android.gms.tasks.Tasks
 import com.google.android.gms.wearable.Wearable
 import kotlinx.coroutines.CoroutineScope
@@ -15,6 +18,10 @@ object MessageSender {
     const val tag = "MessageSender"
     private val job = Job()
     private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
+    var messageLog =   ConcurrentMutableSet<Long>()
+
+
+
 
     fun sendMessage(path: String, message: ByteArray, context: Context) {
         coroutineScope.launch {

wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt

@@ -6,9 +6,13 @@ import android.content.Context
 import android.media.AudioFormat
 import android.media.AudioRecord
 import android.media.MediaRecorder
+import android.os.Message
 import android.util.Log
 import androidx.annotation.RequiresPermission
+import co.touchlab.stately.collections.ConcurrentMutableCollection
 import kotlinx.coroutines.suspendCancellableCoroutine
+import java.nio.ByteBuffer
+import java.time.Instant
 
 /**
  * A helper class to provide methods to record audio input from the MIC to the internal storage.
@@ -27,50 +31,87 @@ class SoundRecorder(
     }
 
 
-
     @RequiresPermission(Manifest.permission.RECORD_AUDIO)
     suspend fun record() {
 
 
         suspendCancellableCoroutine<Unit> { cont ->
+            var chunk_index: Int = 0
             val audioSource = MediaRecorder.AudioSource.DEFAULT
             val sampleRateInHz = 48000
             val channelConfig = AudioFormat.CHANNEL_IN_MONO
-            val audioFormat = AudioFormat.ENCODING_PCM_8BIT
+            val audioFormat = AudioFormat.ENCODING_PCM_16BIT
+            val buffer_size =
+                4 * AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat)
+//            Log.w(TAG, buffer_size.toString())
+
             val bufferSizeInBytes =
-                sampleRateInHz * 1 * 1; // 3 second sample, 2 bytes for each sample
+                sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample
+
+            val chunk_size = 2 * sampleRateInHz / 4  // 250ms segments, 2 bytes for each sample
+            val num_chunks: Int = bufferSizeInBytes / chunk_size
+            val chunked_audio_bytes = Array(num_chunks) { ByteArray(chunk_size) }
+
             val audio_bytes_array = ByteArray(bufferSizeInBytes)
             val audioRecord = AudioRecord(
                 /* audioSource = */ audioSource,
                 /* sampleRateInHz = */ sampleRateInHz,
                 /* channelConfig = */ channelConfig,
                 /* audioFormat = */ audioFormat,
-                /* bufferSizeInBytes = */ bufferSizeInBytes
+                /* bufferSizeInBytes = */ buffer_size
             )
-
+            audioRecord.startRecording()
 
 
             val thread = Thread {
+
+                var last_tstamp: Long = Instant.now().toEpochMilli()
                 while (true) {
+                    if (Thread.interrupted()) {
+                        // check for the interrupted flag, reset it, and throw exception
+                        Log.w(TAG, "Finished thread");
+                        break;
+                    }
+                    chunk_index = chunk_index.mod(num_chunks)
                     val out = audioRecord.read(
-                        /* audioData = */ audio_bytes_array,
+                        /* audioData = */ chunked_audio_bytes[chunk_index],
                         /* offsetInBytes = */ 0,
-                        /* sizeInBytes = */ bufferSizeInBytes,
+                        /* sizeInBytes = */ chunk_size,
-                        /* readMode = */ AudioRecord.READ_BLOCKING
+                        /* readMode = */ android.media.AudioRecord.READ_BLOCKING
                     )
 
-
+                    chunk_index += 1;
-//                    val audio_u_byte = audio_bytes_array.toUByteArray();
+                    if (last_tstamp in MessageSender.messageLog) {
-//                    Log.w(TAG, audio_bytes_array.size.toString());
+                        var tstamp: Long = Instant.now().toEpochMilli()
-                    val str_beg = audio_bytes_array[0].toString()
+                        val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
-                    val str_end = audio_bytes_array[bufferSizeInBytes-1].toString()
+                        val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
-                    Log.w(TAG, str_beg + ",  " + str_end);
+                        var byte_send: ByteArray = ByteArray(0)
-//                    MessageSender.sendMessage("/audio",audio_bytes_array, context)
+                        var strr: String = ""
+                        for (i in 0..(num_chunks - 1)) {
+                            var c_index = i + chunk_index
+                            c_index = c_index.mod(num_chunks)
+                            strr += c_index.toString();
+                            strr += ' '
+                            byte_send += chunked_audio_bytes[c_index]
+                        }
+//                    Log.w(TAG, strr)
+//                    Log.w("MSG_SENT",byte_send.sum().toString()+",.   "+ tstamp.toString())
+                        MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context)
+                        last_tstamp = tstamp;
+                    }
 
                 }
             };
             thread.start();
+//            thread.join();
 
+            cont.invokeOnCancellation {
+                thread.interrupt();
+                audioRecord.stop();
+                audioRecord.release()
+                state = State.IDLE
+            }
+            state = State.IDLE
         }
     }