diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml index 6e6eec1..a55e7a1 100644 --- a/.idea/codeStyles/codeStyleConfig.xml +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -1,6 +1,5 @@ - \ No newline at end of file diff --git a/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt b/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt index a38311c..73dabd4 100644 --- a/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt +++ b/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt @@ -15,6 +15,8 @@ import com.google.android.gms.wearable.Wearable class MainActivity : AppCompatActivity() { // private lateinit var soundClassifier: SoundClassifier + val REQUEST_PERMISSIONS = 1337 + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -28,20 +30,22 @@ class MainActivity : AppCompatActivity() { } } ) - Downloader.downloadModels(this); - requestPermissions(); + Downloader.downloadModels(this) + requestPermissions() soundClassifier = SoundClassifier(this, SoundClassifier.Options()) Location.requestLocation(this, soundClassifier) ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets -> val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars()) v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom) insets + } } companion object { - const val REQUEST_PERMISSIONS = 1337 - @SuppressLint("StaticFieldLeak") - lateinit var soundClassifier: SoundClassifier + var soundClassifier: SoundClassifier? = null +// fun getSoundClassifier(): SoundClassifier? { +// return soundClassifier +// } } private fun requestPermissions() { diff --git a/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt b/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt index 1509e0c..431ecbe 100644 --- a/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt +++ b/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt @@ -18,16 +18,18 @@ class MessageListenerService : WearableListenerService() { override fun onMessageReceived(p0: MessageEvent) { super.onMessageReceived(p0) +// MainActivity val soundclassifier = MainActivity.soundClassifier if (soundclassifier == null) { Log.w(tag, "Have invalid sound classifier") + return } else { Log.w(tag, "Have valid classifier") } - val short_array = ShortArray(48000 * 3); + val short_array = ShortArray(48000 * 3) var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES) var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size) - + var string_send: String = "" ByteBuffer.wrap(audio_bytes).order( ByteOrder.LITTLE_ENDIAN @@ -39,9 +41,13 @@ class MessageListenerService : WearableListenerService() { val score = sorted_list[i].value val index = sorted_list[i].index val species_name = soundclassifier.labelList[index] - Log.w(tag, species_name + ", " + score.toString()); + Log.w(tag, species_name + ", " + score.toString()) + string_send+= species_name + string_send+=',' + string_send+=score.toString() + string_send+=';' } - MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this) + MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this) // Log.i(tag , short_array.map( { abs(it)}).sum().toString()) // Log.i(tag, short_array[0].toString()) // Log.i(tag, p0.data.toString(Charsets.US_ASCII)) diff --git a/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt b/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt index 89993de..c6f1f2c 100644 --- a/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt +++ b/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt @@ -5,6 +5,7 @@ import android.location.Location import android.os.SystemClock import android.preference.PreferenceManager import android.util.Log +import androidx.annotation.Nullable import org.tensorflow.lite.Interpreter import java.io.BufferedReader import java.io.File @@ -26,6 +27,7 @@ import java.nio.ShortBuffer import kotlin.math.round import kotlin.math.sin + class SoundClassifier( context: Context, private val options: Options = Options() @@ -60,10 +62,10 @@ class SoundClassifier( /** Names of the model's output classes. */ - public lateinit var labelList: List + lateinit var labelList: List /** Names of the model's output classes. */ - public lateinit var assetList: List + lateinit var assetList: List /** How many milliseconds between consecutive model inference calls. */ private var inferenceInterval = 800L @@ -315,10 +317,10 @@ class SoundClassifier( private const val NANOS_IN_MILLIS = 1_000_000.toDouble() } - public fun executeScoring( + fun executeScoring( short_array: ShortArray ): List> { - val highPass = 0; + val highPass = 0 val butterworth = Butterworth() butterworth.highPass(6, 48000.0, highPass.toDouble()) @@ -372,7 +374,7 @@ class SoundClassifier( // Load new audio samples // val sampleCounts = loadAudio(recordingBuffer) - val sampleCounts = 0; + val sampleCounts = 0 if (sampleCounts == 0) { return@task } diff --git a/wear/src/main/AndroidManifest.xml b/wear/src/main/AndroidManifest.xml index dc1f106..8f403bd 100644 --- a/wear/src/main/AndroidManifest.xml +++ b/wear/src/main/AndroidManifest.xml @@ -37,7 +37,7 @@ diff --git a/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt b/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt deleted file mode 100644 index a22871f..0000000 --- a/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.birdsounds.identify - -import android.app.Service - - - -import android.content.Intent -import android.util.Half.abs -import android.util.Log -import androidx.localbroadcastmanager.content.LocalBroadcastManager -import com.birdsounds.identify.presentation.MessageSender -import com.google.android.gms.wearable.MessageEvent -import com.google.android.gms.wearable.WearableListenerService -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.nio.ShortBuffer - -class MessageListenerService : WearableListenerService() { - private val tag = "MessageListenerService" - - // fun placeSoundClassifier(soundClassifier: SoundClassifier) - override fun onMessageReceived(p0: MessageEvent) { - super.onMessageReceived(p0) - - - val t_scored = ByteBuffer.wrap(p0.data).getLong() - MessageSender.messageLog.add(t_scored) - Log.w("MSG_RECV", t_scored.toString()) -// Log.i(tag , short_array.map( { abs(it)}).sum().toString()) -// Log.i(tag, short_array[0].toString()) -// Log.i(tag, p0.data.toString(Charsets.US_ASCII)) -// broadcastMessage(p0) - } - -} \ No newline at end of file diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt index f0b3b6e..71f40af 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt @@ -16,13 +16,11 @@ import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.setContent import androidx.activity.result.contract.ActivityResultContracts.RequestPermission import androidx.compose.runtime.Composable -import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.tooling.preview.Preview import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen -import androidx.lifecycle.DefaultLifecycleObserver -import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.viewmodel.compose.viewModel import androidx.wear.compose.navigation.SwipeDismissableNavHost import androidx.wear.compose.navigation.composable @@ -30,11 +28,8 @@ import androidx.wear.compose.navigation.rememberSwipeDismissableNavController import com.birdsounds.identify.presentation.theme.IdentifyTheme import com.google.android.horologist.annotations.ExperimentalHorologistApi import com.google.android.horologist.audio.ui.VolumeViewModel -import com.google.android.horologist.compose.layout.AppScaffold import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.launch -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* val flow_stream = MutableStateFlow("") @@ -63,17 +58,11 @@ class MainActivity : ComponentActivity() { @OptIn(ExperimentalHorologistApi::class) @Composable +@Preview fun WearApp() { IdentifyTheme { lateinit var requestPermissionLauncher: ManagedActivityResultLauncher - -// val job = launch { -// flow_stream.collect { -// print("$it ") -// } -// } - val context = LocalContext.current val activity = context.findActivity() val scope = rememberCoroutineScope() @@ -99,20 +88,20 @@ fun WearApp() { val lifecycleOwner = androidx.lifecycle.compose.LocalLifecycleOwner.current - AppScaffold { - DisposableEffect(mainState, scope, lifecycleOwner) { - val lifecycleObserver = object : DefaultLifecycleObserver { - override fun onStop(owner: LifecycleOwner) { - super.onStop(owner) - } - } - lifecycleOwner.lifecycle.addObserver(lifecycleObserver) - - onDispose { - lifecycleOwner.lifecycle.removeObserver(lifecycleObserver) - } - } - SwipeDismissableNavHost(navController = navController, startDestination = "speaker") { +// AppScaffold { +// DisposableEffect(mainState, scope, lifecycleOwner) { +// val lifecycleObserver = object : DefaultLifecycleObserver { +// override fun onStop(owner: LifecycleOwner) { +// super.onStop(owner) +// } +// } +// lifecycleOwner.lifecycle.addObserver(lifecycleObserver) +// +// onDispose { +// lifecycleOwner.lifecycle.removeObserver(lifecycleObserver) +// } +// } + var sdnv = SwipeDismissableNavHost(navController = navController, startDestination = "speaker") { composable("speaker") { StartRecordingScreen( context = context, @@ -125,7 +114,7 @@ fun WearApp() { }, ) } - } +// } } } } diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt index e265d24..135d40d 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt @@ -46,7 +46,7 @@ class MainState(private val activity: Activity, private val requestPermission: ( when (appState) { is AppState.Ready -> when { (ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) -> { - Log.e(TAG, "Permissions granted, continuing to record"); + Log.e(TAG, "Permissions granted, continuing to record") appState = AppState.Recording @@ -54,7 +54,7 @@ class MainState(private val activity: Activity, private val requestPermission: ( } else -> { - Log.e(TAG, "Requesting permissions"); + Log.e(TAG, "Requesting permissions") requestPermission() } } @@ -100,7 +100,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder, coroutineScope { // Kick off a parallel job to - Log.e(TAG, "Start recording"); // + Log.e(TAG, "Start recording") // val recordingJob = launch { soundRecorder.record() } // val recordingJob = launch { soundRecorder.record() } // SoundRecorder.record(); @@ -142,7 +142,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder, } -object channelCallback : ChannelClient.ChannelCallback() { +object channelCallback : ChannelCallback() { override fun onChannelOpened(channel: ChannelClient.Channel) { super.onChannelOpened(channel) Log.e(TAG,"Opened channel")} diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt new file mode 100644 index 0000000..e585b92 --- /dev/null +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt @@ -0,0 +1,28 @@ +package com.birdsounds.identify.presentation + +import com.google.android.gms.wearable.MessageEvent +import com.google.android.gms.wearable.WearableListenerService +import java.nio.ByteBuffer + + +class MessageListenerService : WearableListenerService() { + private val tag = "MessageListenerService" + + override fun onMessageReceived(p0: MessageEvent) { + super.onMessageReceived(p0) + val t_scored = ByteBuffer.wrap(p0.data).getLong() + var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size) + var score_species_string = byte_strings.decodeToString() + var list_strings: List = score_species_string.split(';') + list_strings.map({ + var split_str = it.split(',') + if (split_str.size == 2) { + var out = AScore(split_str[0], split_str[1].toFloat(), t_scored) + if (out.score > 0.05) { + SpeciesList.add_observation(out) + } + } + }) + MessageSender.messageLog.add(t_scored) + } +} \ No newline at end of file diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt index 69a209d..98025b3 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt @@ -3,10 +3,8 @@ package com.birdsounds.identify.presentation import android.content.Context import android.util.Log -import co.touchlab.stately.collections.ConcurrentMutableCollection -import co.touchlab.stately.collections.ConcurrentMutableList import co.touchlab.stately.collections.ConcurrentMutableSet -import com.google.android.gms.tasks.Tasks +import com.google.android.gms.tasks.Tasksx import com.google.android.gms.wearable.Wearable import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -14,6 +12,7 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.launch import java.util.concurrent.ExecutionException + object MessageSender { const val tag = "MessageSender" private val job = Job() diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt b/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt index 1ae067a..2ed0702 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt @@ -6,10 +6,8 @@ import android.content.Context import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaRecorder -import android.os.Message import android.util.Log import androidx.annotation.RequiresPermission -import co.touchlab.stately.collections.ConcurrentMutableCollection import kotlinx.coroutines.suspendCancellableCoroutine import java.nio.ByteBuffer import java.time.Instant @@ -46,7 +44,7 @@ class SoundRecorder( // Log.w(TAG, buffer_size.toString()) val bufferSizeInBytes = - sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample + sampleRateInHz * 3 * 2 // 3 second sample, 2 bytes for each sample val chunk_size = 2 * sampleRateInHz / 4 // 250ms segments, 2 bytes for each sample val num_chunks: Int = bufferSizeInBytes / chunk_size @@ -62,25 +60,44 @@ class SoundRecorder( ) audioRecord.startRecording() + val thread = Thread { - var num_interactions = 0 +// var sent_first: Boolean = false + var ignore_warmup: Boolean = true + + var num_chunked_since_last_send = 0 var last_tstamp: Long = Instant.now().toEpochMilli() + var do_send_message: Boolean = false while (true) { if (Thread.interrupted()) { // check for the interrupted flag, reset it, and throw exception - Log.w(TAG, "Finished thread"); - break; + Log.w(TAG, "Finished thread") + break } chunk_index = chunk_index.mod(num_chunks) val out = audioRecord.read( /* audioData = */ chunked_audio_bytes[chunk_index], /* offsetInBytes = */ 0, /* sizeInBytes = */ chunk_size, - /* readMode = */ android.media.AudioRecord.READ_BLOCKING + /* readMode = */ AudioRecord.READ_BLOCKING ) + num_chunked_since_last_send += 1 - chunk_index += 1; - if ((last_tstamp in MessageSender.messageLog) || (num_interactions == 0)) { + do_send_message = false + if (num_chunked_since_last_send >= num_chunks) { + do_send_message = true + Log.w("MSG","sending message because full 3s have passed") + } else if ((last_tstamp in MessageSender.messageLog) && (num_chunked_since_last_send>4)) { + do_send_message = true + Log.w("MSG","Send message because the phone has finished") + } else if ((ignore_warmup) && (num_chunked_since_last_send > 2)) { + do_send_message = true + Log.w("MSG","Sent message because ignoring warmup") + } + + + chunk_index += 1 + if ((do_send_message)) { var tstamp: Long = Instant.now().toEpochMilli() val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES) val tstamp_bytes = tstamp_buffer.putLong(tstamp).array() @@ -89,24 +106,25 @@ class SoundRecorder( for (i in 0..(num_chunks - 1)) { var c_index = i + chunk_index c_index = c_index.mod(num_chunks) - strr += c_index.toString(); + strr += c_index.toString() strr += ' ' byte_send += chunked_audio_bytes[c_index] } -// Log.w(TAG, strr) -// Log.w("MSG_SENT",byte_send.sum().toString()+",. "+ tstamp.toString()) +// do_send_message = false; + num_chunked_since_last_send = 0 + MessageSender.messageLog.clear() MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context) - last_tstamp = tstamp; - } + last_tstamp = tstamp + } } - }; - thread.start(); + } + thread.start() // thread.join(); cont.invokeOnCancellation { - thread.interrupt(); - audioRecord.stop(); + thread.interrupt() + audioRecord.stop() audioRecord.release() state = State.IDLE } diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt b/wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt new file mode 100644 index 0000000..b5f6767 --- /dev/null +++ b/wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt @@ -0,0 +1,62 @@ +package com.birdsounds.identify.presentation + +import android.util.Log +import java.time.Instant + + +class AScore( + _species: String, + _score: Float, + _timestamp: Long + +) { + + var split_stuff = _species.split("_") + val species = split_stuff[0] + val score = _score + val common_name = split_stuff[1] + val timestamp = _timestamp + fun age(): Long { + var tstamp: Long = Instant.now().toEpochMilli() + return (tstamp - timestamp) + } + + override fun toString(): String { + var tstamp: Long = Instant.now().toEpochMilli() + return common_name + "," + species + "," + score.toString() + ", " + (age() / 1000.0).toString() + "s ago" + } +} + + +object SpeciesList { + var internal_list = mutableListOf() + var do_add_observation = false + fun add_observation(species_in: AScore) { + do_add_observation = false + var idx = 0 + var idx_replace = -1 + for (i in internal_list) + { + if (i.species == species_in.species) + { + do_add_observation = false + idx_replace = idx + } + idx+=1 + } + if (idx_replace >= 0) + { + Log.w(TAG, "Replacing") + internal_list[idx_replace] = species_in + } else { + internal_list.add(species_in) + } +// val list_reranked = internal_list.withIndex().sortedBy { it -> it.value.age() } + internal_list = internal_list.sortedBy({ (it.age()) }).toMutableList() + + + Log.w(TAG, internal_list.toString()) +// internal_list.add(species_in) + } + +} \ No newline at end of file diff --git a/wear/src/main/res/layout/layout.xml b/wear/src/main/res/layout/layout.xml new file mode 100644 index 0000000..41d2ef1 --- /dev/null +++ b/wear/src/main/res/layout/layout.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file