diff --git a/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt b/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt index 31983d3..a38311c 100644 --- a/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt +++ b/mobile/src/main/java/com/birdsounds/identify/MainActivity.kt @@ -3,6 +3,7 @@ package com.birdsounds.identify import android.content.pm.PackageManager import android.os.Bundle import android.Manifest +import android.annotation.SuppressLint import android.util.Log import androidx.activity.enableEdgeToEdge import androidx.appcompat.app.AppCompatActivity @@ -13,7 +14,7 @@ import com.google.android.gms.wearable.ChannelClient import com.google.android.gms.wearable.Wearable class MainActivity : AppCompatActivity() { - private lateinit var soundClassifier: SoundClassifier +// private lateinit var soundClassifier: SoundClassifier override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -39,6 +40,8 @@ class MainActivity : AppCompatActivity() { } companion object { const val REQUEST_PERMISSIONS = 1337 + @SuppressLint("StaticFieldLeak") + lateinit var soundClassifier: SoundClassifier } private fun requestPermissions() { diff --git a/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt b/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt index 994d7d8..1509e0c 100644 --- a/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt +++ b/mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt @@ -2,17 +2,48 @@ package com.birdsounds.identify import android.content.Intent +import android.util.Half.abs import android.util.Log import androidx.localbroadcastmanager.content.LocalBroadcastManager import com.google.android.gms.wearable.MessageEvent import com.google.android.gms.wearable.WearableListenerService +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.ShortBuffer class MessageListenerService : WearableListenerService() { private val tag = "MessageListenerService" + + // fun placeSoundClassifier(soundClassifier: SoundClassifier) override fun onMessageReceived(p0: MessageEvent) { super.onMessageReceived(p0) -// Log.i(tag ,p0.data.toString(Charsets.UTF_8)) + val soundclassifier = MainActivity.soundClassifier + if (soundclassifier == null) { + Log.w(tag, "Have invalid sound classifier") + } else { + Log.w(tag, "Have valid classifier") + } + val short_array = ShortArray(48000 * 3); + var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES) + var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size) + + + ByteBuffer.wrap(audio_bytes).order( + ByteOrder.LITTLE_ENDIAN + ).asShortBuffer().get(short_array) + Log.w(tag, short_array.sum().toString()) + var sorted_list = soundclassifier.executeScoring(short_array) + Log.w(tag, "") + for (i in 0 until 5) { + val score = sorted_list[i].value + val index = sorted_list[i].index + val species_name = soundclassifier.labelList[index] + Log.w(tag, species_name + ", " + score.toString()); + } + MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this) +// Log.i(tag , short_array.map( { abs(it)}).sum().toString()) +// Log.i(tag, short_array[0].toString()) // Log.i(tag, p0.data.toString(Charsets.US_ASCII)) // broadcastMessage(p0) } diff --git a/mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt b/mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt new file mode 100644 index 0000000..069fdd5 --- /dev/null +++ b/mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt @@ -0,0 +1,57 @@ +package com.birdsounds.identify + +import android.content.Context +import android.util.Log +import com.google.android.gms.tasks.Tasks +import com.google.android.gms.wearable.Wearable +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch +import java.util.concurrent.ExecutionException + +object MessageSenderFromPhone { + const val tag = "MessageSender" + private val job = Job() + private val coroutineScope = CoroutineScope(Dispatchers.IO + job) + + fun sendMessage(path: String, message: ByteArray, context: Context) { + coroutineScope.launch { + sendMessageInBackground(path, message, context) + } + } + + private fun sendMessageInBackground(path: String, message: ByteArray, context: Context) { + //first get all the nodes, ie connected wearable devices. + val nodeListTask = Wearable.getNodeClient(context).connectedNodes + try { + // Block on a task and get the result synchronously (because this is on a background + // thread). + val nodes = Tasks.await(nodeListTask) + if(nodes.isEmpty()) { + Log.i(tag,"No Node found to send message") + } + //Now send the message to each device. + for (node in nodes) { + val sendMessageTask = Wearable.getMessageClient(context) + .sendMessage(node.id, path, message) + try { + // Block on a task and get the result synchronously (because this is on a background + // thread). + val result = Tasks.await(sendMessageTask) + Log.v(tag, "SendThread: message send to " + node.displayName) + } catch (exception: ExecutionException) { + Log.e(tag, "Task failed: $exception") + } catch (exception: InterruptedException) { + Log.e(tag, "Interrupt occurred: $exception") + } + } + } catch (exception: ExecutionException) { + Log.e(tag, "Task failed: $exception") + } catch (exception: InterruptedException) { + Log.e( + tag, "Interrupt occurred: $exception" + ) + } + } +} \ No newline at end of file diff --git a/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt b/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt index bc6f98b..89993de 100644 --- a/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt +++ b/mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt @@ -22,6 +22,7 @@ import kotlin.concurrent.scheduleAtFixedRate import kotlin.math.ceil import kotlin.math.cos import uk.me.berndporr.iirj.Butterworth +import java.nio.ShortBuffer import kotlin.math.round import kotlin.math.sin @@ -31,6 +32,7 @@ class SoundClassifier( ) { internal var mContext: Context val TAG = "Sound Classifier" + init { this.mContext = context.applicationContext } @@ -58,10 +60,10 @@ class SoundClassifier( /** Names of the model's output classes. */ - lateinit var labelList: List + public lateinit var labelList: List /** Names of the model's output classes. */ - lateinit var assetList: List + public lateinit var assetList: List /** How many milliseconds between consecutive model inference calls. */ private var inferenceInterval = 800L @@ -155,6 +157,7 @@ class SoundClassifier( } labelList = wordList.map { it.toTitleCase() } Log.i(TAG, "Label list entries: ${labelList.size}") + } catch (e: IOException) { Log.e(TAG, "Failed to read labels ${filename}: ${e.message}") } @@ -163,12 +166,16 @@ class SoundClassifier( private fun setupInterpreter(context: Context) { try { - val modelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.modelPath + val modelFilePath = context.getDir( + "filesdir", + Context.MODE_PRIVATE + ).absolutePath + "/" + options.modelPath Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath") val modelFile = File(modelFilePath) - val tfliteBuffer: ByteBuffer = FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel -> - channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()) - } + val tfliteBuffer: ByteBuffer = + FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel -> + channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()) + } Log.i(TAG, "Done creating TFLite buffer from $modelFilePath") interpreter = Interpreter(tfliteBuffer, Interpreter.Options()) @@ -201,12 +208,16 @@ class SoundClassifier( private fun setupMetaInterpreter(context: Context) { try { - val metaModelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.metaModelPath + val metaModelFilePath = context.getDir( + "filesdir", + Context.MODE_PRIVATE + ).absolutePath + "/" + options.metaModelPath Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath") val metaModelFile = File(metaModelFilePath) - val tfliteBuffer: ByteBuffer = FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel -> - channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()) - } + val tfliteBuffer: ByteBuffer = + FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel -> + channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()) + } Log.i(TAG, "Done creating TFLite buffer from $metaModelFilePath") meta_interpreter = Interpreter(tfliteBuffer, Interpreter.Options()) @@ -237,7 +248,7 @@ class SoundClassifier( fun runMetaInterpreter(location: Location) { val dayOfYear = LocalDate.now().dayOfYear - val week = ceil( dayOfYear*48.0/366.0) //model year has 48 weeks + val week = ceil(dayOfYear * 48.0 / 366.0) //model year has 48 weeks lat = location.latitude.toFloat() lon = location.longitude.toFloat() @@ -280,6 +291,7 @@ class SoundClassifier( } } + private fun generateDummyAudioInput(inputBuffer: FloatBuffer) { val twoPiTimesFreq = 2 * Math.PI.toFloat() * 1000f for (i in 0 until modelInputLength) { @@ -287,6 +299,7 @@ class SoundClassifier( inputBuffer.put(i, sin(twoPiTimesFreq * x.toDouble()).toFloat()) } } + private fun String.toTitleCase() = splitToSequence("_") .map { it.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.ROOT) else it.toString() } } @@ -297,17 +310,54 @@ class SoundClassifier( private const val TAG = "SoundClassifier" var lat: Float = 0.0f var lon: Float = 0.0f + /** Number of nanoseconds in a millisecond */ private const val NANOS_IN_MILLIS = 1_000_000.toDouble() } + public fun executeScoring( + short_array: ShortArray + ): List> { + val highPass = 0; + val butterworth = Butterworth() + butterworth.highPass(6, 48000.0, highPass.toDouble()) + + val outputBuffer = FloatBuffer.allocate(modelNumClasses) + + for (i in 0 until modelInputLength) { + val s = short_array[i] + + if (highPass == 0) inputBuffer.put(i, s.toFloat()) + else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat()) + } + + + inputBuffer.rewind() + outputBuffer.rewind() + interpreter.run(inputBuffer, outputBuffer) + outputBuffer.rewind() + outputBuffer.get(predictionProbs) // Copy data to predictionProbs. + + val probList = mutableListOf() + for (i in predictionProbs.indices) { + probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid + } + val outList = probList.withIndex().sortedByDescending { it -> it.value } +// Log.w(TAG, outList.toString()) +// Log.i(TAG,labelList[outList[0].index].toString()) + return outList + + + + } + private fun startRecognition() { if (modelInputLength <= 0 || modelNumClasses <= 0) { Log.e(TAG, "Switches: Cannot start recognition because model is unavailable.") return } val sharedPref = PreferenceManager.getDefaultSharedPreferences(mContext) - val highPass = sharedPref.getInt("high_pass",0) + val highPass = sharedPref.getInt("high_pass", 0) val butterworth = Butterworth() butterworth.highPass(6, 48000.0, highPass.toDouble()) @@ -315,7 +365,7 @@ class SoundClassifier( var j = 0 // Indices for the circular buffer next write - Log.w(TAG, "recognitionPeriod:"+inferenceInterval) + Log.w(TAG, "recognitionPeriod:" + inferenceInterval) recognitionTask = Timer().scheduleAtFixedRate(inferenceInterval, inferenceInterval) task@{ val outputBuffer = FloatBuffer.allocate(modelNumClasses) val recordingBuffer = ShortArray(modelInputLength) @@ -341,7 +391,7 @@ class SoundClassifier( if (samplesAreAllZero && s.toInt() != 0) { samplesAreAllZero = false } - if (highPass==0) inputBuffer.put(i, s.toFloat()) + if (highPass == 0) inputBuffer.put(i, s.toFloat()) else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat()) } @@ -365,9 +415,9 @@ class SoundClassifier( // probList.add(1 / (1 + kotlin.math.exp(-value))) //apply sigmoid // } // } else { - for (i in predictionProbs.indices) { - probList.add( metaPredictionProbs[i] / (1+kotlin.math.exp(-predictionProbs[i])) ) //apply sigmoid - } + for (i in predictionProbs.indices) { + probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid + } // } // if (mBinding.progressHorizontal.isIndeterminate){ //if start/stop button set to "running" @@ -375,7 +425,7 @@ class SoundClassifier( // val max = it.maxByOrNull { entry -> entry.value } // updateTextView(max, mBinding.text1) // updateImage(max) - //after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax) + //after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax) // val secondMax = it.filterNot { entry -> entry == max }.maxByOrNull { entry -> entry.value } // updateTextView(secondMax,mBinding.text2) // } diff --git a/wear/build.gradle.kts b/wear/build.gradle.kts index 29dcf5f..862d9e2 100644 --- a/wear/build.gradle.kts +++ b/wear/build.gradle.kts @@ -67,6 +67,7 @@ dependencies { implementation(libs.androidx.core.splashscreen) implementation(libs.androidx.compose.navigation) implementation(libs.androidx.media3.common) + implementation("co.touchlab:stately-concurrent-collections:2.0.0") implementation(libs.androidx.compose.material3) implementation(libs.androidx.work.runtime.ktx) // androidTestImplementation(platform(libs.androidx.compose.bom)) // androidTestImplementation(libs.androidx.ui.test.junit4) diff --git a/wear/src/main/AndroidManifest.xml b/wear/src/main/AndroidManifest.xml index fa063cf..dc1f106 100644 --- a/wear/src/main/AndroidManifest.xml +++ b/wear/src/main/AndroidManifest.xml @@ -34,6 +34,24 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt b/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt new file mode 100644 index 0000000..a22871f --- /dev/null +++ b/wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt @@ -0,0 +1,35 @@ +package com.birdsounds.identify + +import android.app.Service + + + +import android.content.Intent +import android.util.Half.abs +import android.util.Log +import androidx.localbroadcastmanager.content.LocalBroadcastManager +import com.birdsounds.identify.presentation.MessageSender +import com.google.android.gms.wearable.MessageEvent +import com.google.android.gms.wearable.WearableListenerService +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.ShortBuffer + +class MessageListenerService : WearableListenerService() { + private val tag = "MessageListenerService" + + // fun placeSoundClassifier(soundClassifier: SoundClassifier) + override fun onMessageReceived(p0: MessageEvent) { + super.onMessageReceived(p0) + + + val t_scored = ByteBuffer.wrap(p0.data).getLong() + MessageSender.messageLog.add(t_scored) + Log.w("MSG_RECV", t_scored.toString()) +// Log.i(tag , short_array.map( { abs(it)}).sum().toString()) +// Log.i(tag, short_array[0].toString()) +// Log.i(tag, p0.data.toString(Charsets.US_ASCII)) +// broadcastMessage(p0) + } + +} \ No newline at end of file diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt index 524d361..e265d24 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt @@ -1,6 +1,7 @@ package com.birdsounds.identify.presentation import android.Manifest +import android.annotation.SuppressLint import android.app.Activity import android.content.Context import android.content.pm.PackageManager @@ -12,6 +13,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue import androidx.compose.ui.platform.LocalContext import androidx.core.content.ContextCompat +import co.touchlab.stately.collections.ConcurrentMutableCollection import com.google.android.gms.tasks.Tasks import com.google.android.gms.wearable.ChannelClient import com.google.android.gms.wearable.ChannelClient.ChannelCallback @@ -65,6 +67,7 @@ class MainState(private val activity: Activity, private val requestPermission: ( } } + suspend fun permissionResultReturned() { playbackStateMutatorMutex.mutate { if (ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { @@ -136,6 +139,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder, // // Stop recording // recordingJob.cancel() } + } object channelCallback : ChannelClient.ChannelCallback() { diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSendRecv.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MessageSendRecv.kt deleted file mode 100644 index 24ac43a..0000000 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSendRecv.kt +++ /dev/null @@ -1,11 +0,0 @@ -package com.birdsounds.identify.presentation - -import android.content.Context -import androidx.work.CoroutineWorker -import androidx.work.WorkerParameters -import com.google.android.gms.wearable.Wearable - -class MessageSendRecv(private val context: Context) -{ -//MainState. -} \ No newline at end of file diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt b/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt index aa3664b..69a209d 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt @@ -3,6 +3,9 @@ package com.birdsounds.identify.presentation import android.content.Context import android.util.Log +import co.touchlab.stately.collections.ConcurrentMutableCollection +import co.touchlab.stately.collections.ConcurrentMutableList +import co.touchlab.stately.collections.ConcurrentMutableSet import com.google.android.gms.tasks.Tasks import com.google.android.gms.wearable.Wearable import kotlinx.coroutines.CoroutineScope @@ -15,6 +18,10 @@ object MessageSender { const val tag = "MessageSender" private val job = Job() private val coroutineScope = CoroutineScope(Dispatchers.IO + job) + var messageLog = ConcurrentMutableSet() + + + fun sendMessage(path: String, message: ByteArray, context: Context) { coroutineScope.launch { diff --git a/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt b/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt index 99c2e4a..3e1843c 100644 --- a/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt +++ b/wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt @@ -6,9 +6,13 @@ import android.content.Context import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaRecorder +import android.os.Message import android.util.Log import androidx.annotation.RequiresPermission +import co.touchlab.stately.collections.ConcurrentMutableCollection import kotlinx.coroutines.suspendCancellableCoroutine +import java.nio.ByteBuffer +import java.time.Instant /** * A helper class to provide methods to record audio input from the MIC to the internal storage. @@ -27,50 +31,87 @@ class SoundRecorder( } - @RequiresPermission(Manifest.permission.RECORD_AUDIO) suspend fun record() { suspendCancellableCoroutine { cont -> + var chunk_index: Int = 0 val audioSource = MediaRecorder.AudioSource.DEFAULT val sampleRateInHz = 48000 val channelConfig = AudioFormat.CHANNEL_IN_MONO - val audioFormat = AudioFormat.ENCODING_PCM_8BIT + val audioFormat = AudioFormat.ENCODING_PCM_16BIT + val buffer_size = + 4 * AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat) +// Log.w(TAG, buffer_size.toString()) + val bufferSizeInBytes = - sampleRateInHz * 1 * 1; // 3 second sample, 2 bytes for each sample + sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample + + val chunk_size = 2 * sampleRateInHz / 4 // 250ms segments, 2 bytes for each sample + val num_chunks: Int = bufferSizeInBytes / chunk_size + val chunked_audio_bytes = Array(num_chunks) { ByteArray(chunk_size) } + val audio_bytes_array = ByteArray(bufferSizeInBytes) val audioRecord = AudioRecord( /* audioSource = */ audioSource, /* sampleRateInHz = */ sampleRateInHz, /* channelConfig = */ channelConfig, /* audioFormat = */ audioFormat, - /* bufferSizeInBytes = */ bufferSizeInBytes + /* bufferSizeInBytes = */ buffer_size ) - + audioRecord.startRecording() val thread = Thread { + + var last_tstamp: Long = Instant.now().toEpochMilli() while (true) { + if (Thread.interrupted()) { + // check for the interrupted flag, reset it, and throw exception + Log.w(TAG, "Finished thread"); + break; + } + chunk_index = chunk_index.mod(num_chunks) val out = audioRecord.read( - /* audioData = */ audio_bytes_array, + /* audioData = */ chunked_audio_bytes[chunk_index], /* offsetInBytes = */ 0, - /* sizeInBytes = */ bufferSizeInBytes, - /* readMode = */ AudioRecord.READ_BLOCKING + /* sizeInBytes = */ chunk_size, + /* readMode = */ android.media.AudioRecord.READ_BLOCKING ) - -// val audio_u_byte = audio_bytes_array.toUByteArray(); -// Log.w(TAG, audio_bytes_array.size.toString()); - val str_beg = audio_bytes_array[0].toString() - val str_end = audio_bytes_array[bufferSizeInBytes-1].toString() - Log.w(TAG, str_beg + ", " + str_end); -// MessageSender.sendMessage("/audio",audio_bytes_array, context) + chunk_index += 1; + if (last_tstamp in MessageSender.messageLog) { + var tstamp: Long = Instant.now().toEpochMilli() + val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES) + val tstamp_bytes = tstamp_buffer.putLong(tstamp).array() + var byte_send: ByteArray = ByteArray(0) + var strr: String = "" + for (i in 0..(num_chunks - 1)) { + var c_index = i + chunk_index + c_index = c_index.mod(num_chunks) + strr += c_index.toString(); + strr += ' ' + byte_send += chunked_audio_bytes[c_index] + } +// Log.w(TAG, strr) +// Log.w("MSG_SENT",byte_send.sum().toString()+",. "+ tstamp.toString()) + MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context) + last_tstamp = tstamp; + } } }; thread.start(); +// thread.join(); + cont.invokeOnCancellation { + thread.interrupt(); + audioRecord.stop(); + audioRecord.release() + state = State.IDLE + } + state = State.IDLE } }