Bump˚

.idea/codeStyles/codeStyleConfig.xml

@@ -1,6 +1,5 @@
 <component name="ProjectCodeStyleConfiguration">
   <state>
-    <option name="USE_PER_PROJECT_SETTINGS" value="true" />
     <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
   </state>
 </component>

mobile/src/main/java/com/birdsounds/identify/MainActivity.kt

@@ -15,6 +15,8 @@ import com.google.android.gms.wearable.Wearable
 
 class MainActivity : AppCompatActivity() {
 //    private lateinit var soundClassifier: SoundClassifier
+    val REQUEST_PERMISSIONS = 1337
+
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -28,20 +30,22 @@ class MainActivity : AppCompatActivity() {
                 }
             }
             )
-        Downloader.downloadModels(this);
+        Downloader.downloadModels(this)
-        requestPermissions();
+        requestPermissions()
         soundClassifier = SoundClassifier(this, SoundClassifier.Options())
         Location.requestLocation(this, soundClassifier)
         ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets ->
             val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
             v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom)
             insets
+
         }
     }
     companion object {
-        const val REQUEST_PERMISSIONS = 1337
+        var soundClassifier: SoundClassifier? = null
-        @SuppressLint("StaticFieldLeak")
+//        fun getSoundClassifier(): SoundClassifier? {
-        lateinit var soundClassifier: SoundClassifier
+//            return soundClassifier
+//        }
     }
 
     private fun requestPermissions() {

mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt

@@ -18,16 +18,18 @@ class MessageListenerService : WearableListenerService() {
     override fun onMessageReceived(p0: MessageEvent) {
         super.onMessageReceived(p0)
 
+//        MainActivity
         val soundclassifier = MainActivity.soundClassifier
         if (soundclassifier == null) {
             Log.w(tag, "Have invalid sound classifier")
+            return
         } else {
             Log.w(tag, "Have valid classifier")
         }
-        val short_array = ShortArray(48000 * 3);
+        val short_array = ShortArray(48000 * 3)
         var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
         var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
-
+        var string_send: String = ""
 
         ByteBuffer.wrap(audio_bytes).order(
             ByteOrder.LITTLE_ENDIAN
@@ -39,9 +41,13 @@ class MessageListenerService : WearableListenerService() {
             val score = sorted_list[i].value
             val index = sorted_list[i].index
             val species_name = soundclassifier.labelList[index]
-            Log.w(tag, species_name + ", " + score.toString());
+            Log.w(tag, species_name + ", " + score.toString())
+            string_send+= species_name
+            string_send+=','
+            string_send+=score.toString()
+            string_send+=';'
         }
-        MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this)
+        MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
 //        Log.i(tag , short_array.map( { abs(it)}).sum().toString())
 //        Log.i(tag, short_array[0].toString())
 //        Log.i(tag, p0.data.toString(Charsets.US_ASCII))

mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt

@@ -5,6 +5,7 @@ import android.location.Location
 import android.os.SystemClock
 import android.preference.PreferenceManager
 import android.util.Log
+import androidx.annotation.Nullable
 import org.tensorflow.lite.Interpreter
 import java.io.BufferedReader
 import java.io.File
@@ -26,6 +27,7 @@ import java.nio.ShortBuffer
 import kotlin.math.round
 import kotlin.math.sin
 
+
 class SoundClassifier(
     context: Context,
     private val options: Options = Options()
@@ -60,10 +62,10 @@ class SoundClassifier(
 
 
     /** Names of the model's output classes.  */
-    public lateinit var labelList: List<String>
+    lateinit var labelList: List<String>
 
     /** Names of the model's output classes.  */
-    public lateinit var assetList: List<String>
+    lateinit var assetList: List<String>
 
     /** How many milliseconds between consecutive model inference calls.  */
     private var inferenceInterval = 800L
@@ -315,10 +317,10 @@ class SoundClassifier(
         private const val NANOS_IN_MILLIS = 1_000_000.toDouble()
     }
 
-    public fun executeScoring(
+    fun executeScoring(
         short_array: ShortArray
     ): List<IndexedValue<Float>> {
-        val highPass = 0;
+        val highPass = 0
         val butterworth = Butterworth()
         butterworth.highPass(6, 48000.0, highPass.toDouble())
 
@@ -372,7 +374,7 @@ class SoundClassifier(
 
             // Load new audio samples
 //            val sampleCounts = loadAudio(recordingBuffer)
-            val sampleCounts = 0;
+            val sampleCounts = 0
             if (sampleCounts == 0) {
                 return@task
             }

wear/src/main/AndroidManifest.xml

@@ -37,7 +37,7 @@
 
 
         <service
-            android:name=".MessageListenerService"
+            android:name=".presentation.MessageListenerService"
             android:enabled="true"
             android:exported="true" >
             <intent-filter>

wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt

@@ -1,35 +0,0 @@
-package com.birdsounds.identify
-
-import android.app.Service
-
-
-
-import android.content.Intent
-import android.util.Half.abs
-import android.util.Log
-import androidx.localbroadcastmanager.content.LocalBroadcastManager
-import com.birdsounds.identify.presentation.MessageSender
-import com.google.android.gms.wearable.MessageEvent
-import com.google.android.gms.wearable.WearableListenerService
-import java.nio.ByteBuffer
-import java.nio.ByteOrder
-import java.nio.ShortBuffer
-
-class MessageListenerService : WearableListenerService() {
-    private val tag = "MessageListenerService"
-
-    //    fun placeSoundClassifier(soundClassifier: SoundClassifier)
-    override fun onMessageReceived(p0: MessageEvent) {
-        super.onMessageReceived(p0)
-
-
-        val t_scored = ByteBuffer.wrap(p0.data).getLong()
-        MessageSender.messageLog.add(t_scored)
-        Log.w("MSG_RECV", t_scored.toString())
-//        Log.i(tag , short_array.map( { abs(it)}).sum().toString())
-//        Log.i(tag, short_array[0].toString())
-//        Log.i(tag, p0.data.toString(Charsets.US_ASCII))
-//        broadcastMessage(p0)
-    }
-
-}

wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt

@@ -16,13 +16,11 @@ import androidx.activity.compose.rememberLauncherForActivityResult
 import androidx.activity.compose.setContent
 import androidx.activity.result.contract.ActivityResultContracts.RequestPermission
 import androidx.compose.runtime.Composable
-import androidx.compose.runtime.DisposableEffect
 import androidx.compose.runtime.remember
 import androidx.compose.runtime.rememberCoroutineScope
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.tooling.preview.Preview
 import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen
-import androidx.lifecycle.DefaultLifecycleObserver
-import androidx.lifecycle.LifecycleOwner
 import androidx.lifecycle.viewmodel.compose.viewModel
 import androidx.wear.compose.navigation.SwipeDismissableNavHost
 import androidx.wear.compose.navigation.composable
@@ -30,11 +28,8 @@ import androidx.wear.compose.navigation.rememberSwipeDismissableNavController
 import com.birdsounds.identify.presentation.theme.IdentifyTheme
 import com.google.android.horologist.annotations.ExperimentalHorologistApi
 import com.google.android.horologist.audio.ui.VolumeViewModel
-import com.google.android.horologist.compose.layout.AppScaffold
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.*
 
 val flow_stream = MutableStateFlow<String>("")
 
@@ -63,17 +58,11 @@ class MainActivity : ComponentActivity() {
 
 @OptIn(ExperimentalHorologistApi::class)
 @Composable
+@Preview
 fun WearApp() {
 
     IdentifyTheme {
         lateinit var requestPermissionLauncher: ManagedActivityResultLauncher<String, Boolean>
-
-//        val job = launch {
-//            flow_stream.collect {
-//                print("$it ")
-//            }
-//        }
-
         val context = LocalContext.current
         val activity = context.findActivity()
         val scope = rememberCoroutineScope()
@@ -99,20 +88,20 @@ fun WearApp() {
 
         val lifecycleOwner = androidx.lifecycle.compose.LocalLifecycleOwner.current
 
-        AppScaffold {
+//        AppScaffold {
-            DisposableEffect(mainState, scope, lifecycleOwner) {
+//            DisposableEffect(mainState, scope, lifecycleOwner) {
-                val lifecycleObserver = object : DefaultLifecycleObserver {
+//                val lifecycleObserver = object : DefaultLifecycleObserver {
-                    override fun onStop(owner: LifecycleOwner) {
+//                    override fun onStop(owner: LifecycleOwner) {
-                        super.onStop(owner)
+//                        super.onStop(owner)
-                    }
+//                    }
-                }
+//                }
-                lifecycleOwner.lifecycle.addObserver(lifecycleObserver)
+//                lifecycleOwner.lifecycle.addObserver(lifecycleObserver)
-
+//
-                onDispose {
+//                onDispose {
-                    lifecycleOwner.lifecycle.removeObserver(lifecycleObserver)
+//                    lifecycleOwner.lifecycle.removeObserver(lifecycleObserver)
-                }
+//                }
-            }
+//            }
-            SwipeDismissableNavHost(navController = navController, startDestination = "speaker") {
+            var sdnv = SwipeDismissableNavHost(navController = navController, startDestination = "speaker") {
                 composable("speaker") {
                     StartRecordingScreen(
                         context = context,
@@ -125,7 +114,7 @@ fun WearApp() {
                         },
                     )
                 }
-            }
+//            }
         }
     }
 }

wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt

@@ -46,7 +46,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
             when (appState) {
                 is AppState.Ready  -> when {
                     (ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) -> {
-                        Log.e(TAG, "Permissions granted, continuing to record");
+                        Log.e(TAG, "Permissions granted, continuing to record")
                         appState = AppState.Recording
 
 
@@ -54,7 +54,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
                     }
 
                     else                                                                                                                 -> {
-                        Log.e(TAG, "Requesting permissions");
+                        Log.e(TAG, "Requesting permissions")
                         requestPermission()
                     }
                 }
@@ -100,7 +100,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
 
     coroutineScope { // Kick off a parallel job to
 
-        Log.e(TAG, "Start recording"); //
+        Log.e(TAG, "Start recording") //
         val recordingJob = launch { soundRecorder.record() }
 //    val recordingJob = launch { soundRecorder.record() }
 //        SoundRecorder.record();
@@ -142,7 +142,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
 
 }
 
-object channelCallback : ChannelClient.ChannelCallback() {
+object channelCallback : ChannelCallback() {
     override fun onChannelOpened(channel: ChannelClient.Channel) {
         super.onChannelOpened(channel)
     Log.e(TAG,"Opened channel")}

wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt

@@ -0,0 +1,28 @@
+package com.birdsounds.identify.presentation
+
+import com.google.android.gms.wearable.MessageEvent
+import com.google.android.gms.wearable.WearableListenerService
+import java.nio.ByteBuffer
+
+
+class MessageListenerService : WearableListenerService() {
+    private val tag = "MessageListenerService"
+
+    override fun onMessageReceived(p0: MessageEvent) {
+        super.onMessageReceived(p0)
+        val t_scored = ByteBuffer.wrap(p0.data).getLong()
+        var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
+        var score_species_string = byte_strings.decodeToString()
+        var list_strings: List<String> = score_species_string.split(';')
+        list_strings.map({
+            var split_str = it.split(',')
+            if (split_str.size == 2) {
+                var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
+                if (out.score > 0.05) {
+                    SpeciesList.add_observation(out)
+                }
+            }
+        })
+        MessageSender.messageLog.add(t_scored)
+    }
+}

wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt

@@ -3,10 +3,8 @@ package com.birdsounds.identify.presentation
 
 import android.content.Context
 import android.util.Log
-import co.touchlab.stately.collections.ConcurrentMutableCollection
-import co.touchlab.stately.collections.ConcurrentMutableList
 import co.touchlab.stately.collections.ConcurrentMutableSet
-import com.google.android.gms.tasks.Tasks
+import com.google.android.gms.tasks.Tasksx
 import com.google.android.gms.wearable.Wearable
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -14,6 +12,7 @@ import kotlinx.coroutines.Job
 import kotlinx.coroutines.launch
 import java.util.concurrent.ExecutionException
 
+
 object MessageSender {
     const val tag = "MessageSender"
     private val job = Job()

wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt

@@ -6,10 +6,8 @@ import android.content.Context
 import android.media.AudioFormat
 import android.media.AudioRecord
 import android.media.MediaRecorder
-import android.os.Message
 import android.util.Log
 import androidx.annotation.RequiresPermission
-import co.touchlab.stately.collections.ConcurrentMutableCollection
 import kotlinx.coroutines.suspendCancellableCoroutine
 import java.nio.ByteBuffer
 import java.time.Instant
@@ -46,7 +44,7 @@ class SoundRecorder(
 //            Log.w(TAG, buffer_size.toString())
 
             val bufferSizeInBytes =
-                sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample
+                sampleRateInHz * 3 * 2 // 3 second sample, 2 bytes for each sample
 
             val chunk_size = 2 * sampleRateInHz / 4  // 250ms segments, 2 bytes for each sample
             val num_chunks: Int = bufferSizeInBytes / chunk_size
@@ -62,25 +60,44 @@ class SoundRecorder(
             )
             audioRecord.startRecording()
 
+
             val thread = Thread {
-                var num_interactions = 0
+//                var sent_first: Boolean = false
+                var ignore_warmup: Boolean = true
+
+                var num_chunked_since_last_send = 0
                 var last_tstamp: Long = Instant.now().toEpochMilli()
+                var do_send_message: Boolean = false
                 while (true) {
                     if (Thread.interrupted()) {
                         // check for the interrupted flag, reset it, and throw exception
-                        Log.w(TAG, "Finished thread");
+                        Log.w(TAG, "Finished thread")
-                        break;
+                        break
                     }
                     chunk_index = chunk_index.mod(num_chunks)
                     val out = audioRecord.read(
                         /* audioData = */ chunked_audio_bytes[chunk_index],
                         /* offsetInBytes = */ 0,
                         /* sizeInBytes = */ chunk_size,
-                        /* readMode = */ android.media.AudioRecord.READ_BLOCKING
+                        /* readMode = */ AudioRecord.READ_BLOCKING
                     )
+                    num_chunked_since_last_send += 1
 
-                    chunk_index += 1;
+                    do_send_message = false
-                    if ((last_tstamp in MessageSender.messageLog) || (num_interactions == 0)) {
+                    if (num_chunked_since_last_send >= num_chunks) {
+                        do_send_message = true
+                        Log.w("MSG","sending message because full 3s have passed")
+                    } else if ((last_tstamp in MessageSender.messageLog) && (num_chunked_since_last_send>4)) {
+                        do_send_message = true
+                        Log.w("MSG","Send message because the phone has finished")
+                    } else if ((ignore_warmup) && (num_chunked_since_last_send > 2)) {
+                        do_send_message = true
+                        Log.w("MSG","Sent message because ignoring warmup")
+                    }
+
+
+                    chunk_index += 1
+                    if ((do_send_message)) {
                         var tstamp: Long = Instant.now().toEpochMilli()
                         val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
                         val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
@@ -89,24 +106,25 @@ class SoundRecorder(
                         for (i in 0..(num_chunks - 1)) {
                             var c_index = i + chunk_index
                             c_index = c_index.mod(num_chunks)
-                            strr += c_index.toString();
+                            strr += c_index.toString()
                             strr += ' '
                             byte_send += chunked_audio_bytes[c_index]
                         }
-//                    Log.w(TAG, strr)
+//                        do_send_message = false;
-//                    Log.w("MSG_SENT",byte_send.sum().toString()+",.   "+ tstamp.toString())
+                        num_chunked_since_last_send = 0
+                        MessageSender.messageLog.clear()
                         MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context)
-                        last_tstamp = tstamp;
+                        last_tstamp = tstamp
-                    }
 
                     }
-            };
+                }
-            thread.start();
+            }
+            thread.start()
 //            thread.join();
 
             cont.invokeOnCancellation {
-                thread.interrupt();
+                thread.interrupt()
-                audioRecord.stop();
+                audioRecord.stop()
                 audioRecord.release()
                 state = State.IDLE
             }

wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt

@@ -0,0 +1,62 @@
+package com.birdsounds.identify.presentation
+
+import android.util.Log
+import java.time.Instant
+
+
+class AScore(
+    _species: String,
+    _score: Float,
+    _timestamp: Long
+
+) {
+
+    var split_stuff = _species.split("_")
+    val species = split_stuff[0]
+    val score = _score
+    val common_name = split_stuff[1]
+    val timestamp = _timestamp
+    fun age(): Long {
+        var tstamp: Long = Instant.now().toEpochMilli()
+        return (tstamp - timestamp)
+    }
+
+    override fun toString(): String {
+        var tstamp: Long = Instant.now().toEpochMilli()
+        return common_name + "," + species + "," + score.toString() + ", " + (age() / 1000.0).toString() + "s ago"
+    }
+}
+
+
+object SpeciesList {
+    var internal_list = mutableListOf<AScore>()
+    var do_add_observation = false
+    fun add_observation(species_in: AScore) {
+        do_add_observation = false
+        var idx = 0
+        var idx_replace = -1
+        for (i in internal_list)
+        {
+            if (i.species == species_in.species)
+            {
+                do_add_observation = false
+                idx_replace = idx
+            }
+            idx+=1
+        }
+        if (idx_replace >= 0)
+        {
+            Log.w(TAG, "Replacing")
+            internal_list[idx_replace] = species_in
+        } else {
+            internal_list.add(species_in)
+        }
+//        val list_reranked = internal_list.withIndex().sortedBy { it -> it.value.age() }
+        internal_list = internal_list.sortedBy({ (it.age()) }).toMutableList()
+
+
+        Log.w(TAG, internal_list.toString())
+//        internal_list.add(species_in)
+    }
+
+}

wear/src/main/res/layout/layout.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+</LinearLayout>