thebears/bird_sound_identify_wearos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump

Browse Source
This commit is contained in:
isp
2024-09-10 19:30:49 -07:00
parent c04123ea84
commit da7f342994
11 changed files with 282 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
mobile/src/main/java/com/birdsounds/identify/MainActivity.kt
View File

@@ -3,6 +3,7 @@ package com.birdsounds.identify
import android.content.pm.PackageManager
import android.os.Bundle
import android.Manifest
import android.annotation.SuppressLint
import android.util.Log
import androidx.activity.enableEdgeToEdge
import androidx.appcompat.app.AppCompatActivity
@@ -13,7 +14,7 @@ import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
class MainActivity : AppCompatActivity() {
private lateinit var soundClassifier: SoundClassifier
// private lateinit var soundClassifier: SoundClassifier
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
@@ -39,6 +40,8 @@ class MainActivity : AppCompatActivity() {
}
companion object {
const val REQUEST_PERMISSIONS = 1337
@SuppressLint("StaticFieldLeak")
lateinit var soundClassifier: SoundClassifier
}
private fun requestPermissions() {

33
mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt
View File

@@ -2,17 +2,48 @@ package com.birdsounds.identify
import android.content.Intent
import android.util.Half.abs
import android.util.Log
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.ShortBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// Log.i(tag ,p0.data.toString(Charsets.UTF_8))
val soundclassifier = MainActivity.soundClassifier
if (soundclassifier == null) {
Log.w(tag, "Have invalid sound classifier")
} else {
Log.w(tag, "Have valid classifier")
}
val short_array = ShortArray(48000 * 3);
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array)
Log.w(tag, short_array.sum().toString())
var sorted_list = soundclassifier.executeScoring(short_array)
Log.w(tag, "")
for (i in 0 until 5) {
val score = sorted_list[i].value
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(tag, species_name + ", " + score.toString());
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this)
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))
// broadcastMessage(p0)
}

57
mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt Normal file
View File

@@ -0,0 +1,57 @@
package com.birdsounds.identify
import android.content.Context
import android.util.Log
import com.google.android.gms.tasks.Tasks
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import java.util.concurrent.ExecutionException
object MessageSenderFromPhone {
const val tag = "MessageSender"
private val job = Job()
private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
fun sendMessage(path: String, message: ByteArray, context: Context) {
coroutineScope.launch {
sendMessageInBackground(path, message, context)
}
}
private fun sendMessageInBackground(path: String, message: ByteArray, context: Context) {
//first get all the nodes, ie connected wearable devices.
val nodeListTask = Wearable.getNodeClient(context).connectedNodes
try {
// Block on a task and get the result synchronously (because this is on a background
// thread).
val nodes = Tasks.await(nodeListTask)
if(nodes.isEmpty()) {
Log.i(tag,"No Node found to send message")
}
//Now send the message to each device.
for (node in nodes) {
val sendMessageTask = Wearable.getMessageClient(context)
.sendMessage(node.id, path, message)
try {
// Block on a task and get the result synchronously (because this is on a background
// thread).
val result = Tasks.await(sendMessageTask)
Log.v(tag, "SendThread: message send to " + node.displayName)
} catch (exception: ExecutionException) {
Log.e(tag, "Task failed: $exception")
} catch (exception: InterruptedException) {
Log.e(tag, "Interrupt occurred: $exception")
}
}
} catch (exception: ExecutionException) {
Log.e(tag, "Task failed: $exception")
} catch (exception: InterruptedException) {
Log.e(
tag, "Interrupt occurred: $exception"
)
}
}
}

72
mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt
View File

@@ -22,6 +22,7 @@ import kotlin.concurrent.scheduleAtFixedRate
import kotlin.math.ceil
import kotlin.math.cos
import uk.me.berndporr.iirj.Butterworth
import java.nio.ShortBuffer
import kotlin.math.round
import kotlin.math.sin
@@ -31,6 +32,7 @@ class SoundClassifier(
) {
internal var mContext: Context
val TAG = "Sound Classifier"
init {
this.mContext = context.applicationContext
}
@@ -58,10 +60,10 @@ class SoundClassifier(
/** Names of the model's output classes. */
lateinit var labelList: List<String>
public lateinit var labelList: List<String>
/** Names of the model's output classes. */
lateinit var assetList: List<String>
public lateinit var assetList: List<String>
/** How many milliseconds between consecutive model inference calls. */
private var inferenceInterval = 800L
@@ -155,6 +157,7 @@ class SoundClassifier(
}
labelList = wordList.map { it.toTitleCase() }
Log.i(TAG, "Label list entries: ${labelList.size}")
} catch (e: IOException) {
Log.e(TAG, "Failed to read labels ${filename}: ${e.message}")
}
@@ -163,10 +166,14 @@ class SoundClassifier(
private fun setupInterpreter(context: Context) {
try {
val modelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.modelPath
val modelFilePath = context.getDir(
"filesdir",
Context.MODE_PRIVATE
).absolutePath + "/" + options.modelPath
Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
val modelFile = File(modelFilePath)
val tfliteBuffer: ByteBuffer = FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
val tfliteBuffer: ByteBuffer =
FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
Log.i(TAG, "Done creating TFLite buffer from $modelFilePath")
@@ -201,10 +208,14 @@ class SoundClassifier(
private fun setupMetaInterpreter(context: Context) {
try {
val metaModelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.metaModelPath
val metaModelFilePath = context.getDir(
"filesdir",
Context.MODE_PRIVATE
).absolutePath + "/" + options.metaModelPath
Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath")
val metaModelFile = File(metaModelFilePath)
val tfliteBuffer: ByteBuffer = FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
val tfliteBuffer: ByteBuffer =
FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
Log.i(TAG, "Done creating TFLite buffer from $metaModelFilePath")
@@ -237,7 +248,7 @@ class SoundClassifier(
fun runMetaInterpreter(location: Location) {
val dayOfYear = LocalDate.now().dayOfYear
val week = ceil( dayOfYear*48.0/366.0) //model year has 48 weeks
val week = ceil(dayOfYear * 48.0 / 366.0) //model year has 48 weeks
lat = location.latitude.toFloat()
lon = location.longitude.toFloat()
@@ -280,6 +291,7 @@ class SoundClassifier(
}
}
private fun generateDummyAudioInput(inputBuffer: FloatBuffer) {
val twoPiTimesFreq = 2 * Math.PI.toFloat() * 1000f
for (i in 0 until modelInputLength) {
@@ -287,6 +299,7 @@ class SoundClassifier(
inputBuffer.put(i, sin(twoPiTimesFreq * x.toDouble()).toFloat())
}
}
private fun String.toTitleCase() =
splitToSequence("_")
.map { it.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.ROOT) else it.toString() } }
@@ -297,17 +310,54 @@ class SoundClassifier(
private const val TAG = "SoundClassifier"
var lat: Float = 0.0f
var lon: Float = 0.0f
/** Number of nanoseconds in a millisecond */
private const val NANOS_IN_MILLIS = 1_000_000.toDouble()
}
public fun executeScoring(
short_array: ShortArray
): List<IndexedValue<Float>> {
val highPass = 0;
val butterworth = Butterworth()
butterworth.highPass(6, 48000.0, highPass.toDouble())
val outputBuffer = FloatBuffer.allocate(modelNumClasses)
for (i in 0 until modelInputLength) {
val s = short_array[i]
if (highPass == 0) inputBuffer.put(i, s.toFloat())
else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
}
inputBuffer.rewind()
outputBuffer.rewind()
interpreter.run(inputBuffer, outputBuffer)
outputBuffer.rewind()
outputBuffer.get(predictionProbs) // Copy data to predictionProbs.
val probList = mutableListOf<Float>()
for (i in predictionProbs.indices) {
probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid
}
val outList = probList.withIndex().sortedByDescending { it -> it.value }
// Log.w(TAG, outList.toString())
// Log.i(TAG,labelList[outList[0].index].toString())
return outList
}
private fun startRecognition() {
if (modelInputLength <= 0 || modelNumClasses <= 0) {
Log.e(TAG, "Switches: Cannot start recognition because model is unavailable.")
return
}
val sharedPref = PreferenceManager.getDefaultSharedPreferences(mContext)
val highPass = sharedPref.getInt("high_pass",0)
val highPass = sharedPref.getInt("high_pass", 0)
val butterworth = Butterworth()
butterworth.highPass(6, 48000.0, highPass.toDouble())
@@ -315,7 +365,7 @@ class SoundClassifier(
var j = 0 // Indices for the circular buffer next write
Log.w(TAG, "recognitionPeriod:"+inferenceInterval)
Log.w(TAG, "recognitionPeriod:" + inferenceInterval)
recognitionTask = Timer().scheduleAtFixedRate(inferenceInterval, inferenceInterval) task@{
val outputBuffer = FloatBuffer.allocate(modelNumClasses)
val recordingBuffer = ShortArray(modelInputLength)
@@ -341,7 +391,7 @@ class SoundClassifier(
if (samplesAreAllZero && s.toInt() != 0) {
samplesAreAllZero = false
}
if (highPass==0) inputBuffer.put(i, s.toFloat())
if (highPass == 0) inputBuffer.put(i, s.toFloat())
else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
}
@@ -366,7 +416,7 @@ class SoundClassifier(
// }
// } else {
for (i in predictionProbs.indices) {
probList.add( metaPredictionProbs[i] / (1+kotlin.math.exp(-predictionProbs[i])) ) //apply sigmoid
probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid
}
// }

1
wear/build.gradle.kts
View File

@@ -67,6 +67,7 @@ dependencies {
implementation(libs.androidx.core.splashscreen)
implementation(libs.androidx.compose.navigation)
implementation(libs.androidx.media3.common)
implementation("co.touchlab:stately-concurrent-collections:2.0.0")
implementation(libs.androidx.compose.material3)
implementation(libs.androidx.work.runtime.ktx) // androidTestImplementation(platform(libs.androidx.compose.bom))
// androidTestImplementation(libs.androidx.ui.test.junit4)

18
wear/src/main/AndroidManifest.xml
View File

@@ -34,6 +34,24 @@
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<service
android:name=".MessageListenerService"
android:enabled="true"
android:exported="true" >
<intent-filter>
<action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
<data
android:host="*"
android:pathPrefix="/audio"
android:scheme="wear" />
</intent-filter>
</service>
</application>
</manifest>

35
wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt Normal file
View File

@@ -0,0 +1,35 @@
package com.birdsounds.identify
import android.app.Service
import android.content.Intent
import android.util.Half.abs
import android.util.Log
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import com.birdsounds.identify.presentation.MessageSender
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.ShortBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
val t_scored = ByteBuffer.wrap(p0.data).getLong()
MessageSender.messageLog.add(t_scored)
Log.w("MSG_RECV", t_scored.toString())
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))
// broadcastMessage(p0)
}
}

4
wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt
View File

@@ -1,6 +1,7 @@
package com.birdsounds.identify.presentation
import android.Manifest
import android.annotation.SuppressLint
import android.app.Activity
import android.content.Context
import android.content.pm.PackageManager
@@ -12,6 +13,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.compose.ui.platform.LocalContext
import androidx.core.content.ContextCompat
import co.touchlab.stately.collections.ConcurrentMutableCollection
import com.google.android.gms.tasks.Tasks
import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.ChannelClient.ChannelCallback
@@ -65,6 +67,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
}
}
suspend fun permissionResultReturned() {
playbackStateMutatorMutex.mutate {
if (ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
@@ -136,6 +139,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
// // Stop recording
// recordingJob.cancel()
}
}
object channelCallback : ChannelClient.ChannelCallback() {

11
wear/src/main/java/com/birdsounds/identify/presentation/MessageSendRecv.kt
View File

@@ -1,11 +0,0 @@
package com.birdsounds.identify.presentation
import android.content.Context
import androidx.work.CoroutineWorker
import androidx.work.WorkerParameters
import com.google.android.gms.wearable.Wearable
class MessageSendRecv(private val context: Context)
{
//MainState.
}

7
wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt
View File

@@ -3,6 +3,9 @@ package com.birdsounds.identify.presentation
import android.content.Context
import android.util.Log
import co.touchlab.stately.collections.ConcurrentMutableCollection
import co.touchlab.stately.collections.ConcurrentMutableList
import co.touchlab.stately.collections.ConcurrentMutableSet
import com.google.android.gms.tasks.Tasks
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CoroutineScope
@@ -15,6 +18,10 @@ object MessageSender {
const val tag = "MessageSender"
private val job = Job()
private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
var messageLog = ConcurrentMutableSet<Long>()
fun sendMessage(path: String, message: ByteArray, context: Context) {
coroutineScope.launch {

71
wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt
View File

@@ -6,9 +6,13 @@ import android.content.Context
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.os.Message
import android.util.Log
import androidx.annotation.RequiresPermission
import co.touchlab.stately.collections.ConcurrentMutableCollection
import kotlinx.coroutines.suspendCancellableCoroutine
import java.nio.ByteBuffer
import java.time.Instant
/**
* A helper class to provide methods to record audio input from the MIC to the internal storage.
@@ -27,50 +31,87 @@ class SoundRecorder(
}
@RequiresPermission(Manifest.permission.RECORD_AUDIO)
suspend fun record() {
suspendCancellableCoroutine<Unit> { cont ->
var chunk_index: Int = 0
val audioSource = MediaRecorder.AudioSource.DEFAULT
val sampleRateInHz = 48000
val channelConfig = AudioFormat.CHANNEL_IN_MONO
val audioFormat = AudioFormat.ENCODING_PCM_8BIT
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
val buffer_size =
4 * AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat)
// Log.w(TAG, buffer_size.toString())
val bufferSizeInBytes =
sampleRateInHz * 1 * 1; // 3 second sample, 2 bytes for each sample
sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample
val chunk_size = 2 * sampleRateInHz / 4 // 250ms segments, 2 bytes for each sample
val num_chunks: Int = bufferSizeInBytes / chunk_size
val chunked_audio_bytes = Array(num_chunks) { ByteArray(chunk_size) }
val audio_bytes_array = ByteArray(bufferSizeInBytes)
val audioRecord = AudioRecord(
/* audioSource = */ audioSource,
/* sampleRateInHz = */ sampleRateInHz,
/* channelConfig = */ channelConfig,
/* audioFormat = */ audioFormat,
/* bufferSizeInBytes = */ bufferSizeInBytes
/* bufferSizeInBytes = */ buffer_size
)
audioRecord.startRecording()
val thread = Thread {
var last_tstamp: Long = Instant.now().toEpochMilli()
while (true) {
if (Thread.interrupted()) {
// check for the interrupted flag, reset it, and throw exception
Log.w(TAG, "Finished thread");
break;
}
chunk_index = chunk_index.mod(num_chunks)
val out = audioRecord.read(
/* audioData = */ audio_bytes_array,
/* audioData = */ chunked_audio_bytes[chunk_index],
/* offsetInBytes = */ 0,
/* sizeInBytes = */ bufferSizeInBytes,
/* readMode = */ AudioRecord.READ_BLOCKING
/* sizeInBytes = */ chunk_size,
/* readMode = */ android.media.AudioRecord.READ_BLOCKING
)
// val audio_u_byte = audio_bytes_array.toUByteArray();
// Log.w(TAG, audio_bytes_array.size.toString());
val str_beg = audio_bytes_array[0].toString()
val str_end = audio_bytes_array[bufferSizeInBytes-1].toString()
Log.w(TAG, str_beg + ", " + str_end);
// MessageSender.sendMessage("/audio",audio_bytes_array, context)
chunk_index += 1;
if (last_tstamp in MessageSender.messageLog) {
var tstamp: Long = Instant.now().toEpochMilli()
val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
var byte_send: ByteArray = ByteArray(0)
var strr: String = ""
for (i in 0..(num_chunks - 1)) {
var c_index = i + chunk_index
c_index = c_index.mod(num_chunks)
strr += c_index.toString();
strr += ' '
byte_send += chunked_audio_bytes[c_index]
}
// Log.w(TAG, strr)
// Log.w("MSG_SENT",byte_send.sum().toString()+",. "+ tstamp.toString())
MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context)
last_tstamp = tstamp;
}
}
};
thread.start();
// thread.join();
cont.invokeOnCancellation {
thread.interrupt();
audioRecord.stop();
audioRecord.release()
state = State.IDLE
}
state = State.IDLE
}
}