thebears/bird_sound_identify_wearos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump

Browse Source
This commit is contained in:
isp
2024-09-10 19:30:49 -07:00
parent c04123ea84
commit da7f342994
11 changed files with 282 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
mobile/src/main/java/com/birdsounds/identify/MainActivity.kt
View File

@@ -3,6 +3,7 @@ package com.birdsounds.identify
import android.content.pm.PackageManager
import android.os.Bundle
import android.Manifest
import android.annotation.SuppressLint
import android.util.Log
import androidx.activity.enableEdgeToEdge
import androidx.appcompat.app.AppCompatActivity
@@ -13,7 +14,7 @@ import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
class MainActivity : AppCompatActivity() {
private lateinit var soundClassifier: SoundClassifier
// private lateinit var soundClassifier: SoundClassifier
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
@@ -39,6 +40,8 @@ class MainActivity : AppCompatActivity() {
}
companion object {
const val REQUEST_PERMISSIONS = 1337
@SuppressLint("StaticFieldLeak")
lateinit var soundClassifier: SoundClassifier
}
private fun requestPermissions() {

33
mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt
View File

@@ -2,17 +2,48 @@ package com.birdsounds.identify
import android.content.Intent
import android.util.Half.abs
import android.util.Log
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.ShortBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// Log.i(tag ,p0.data.toString(Charsets.UTF_8))
val soundclassifier = MainActivity.soundClassifier
if (soundclassifier == null) {
Log.w(tag, "Have invalid sound classifier")
} else {
Log.w(tag, "Have valid classifier")
}
val short_array = ShortArray(48000 * 3);
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array)
Log.w(tag, short_array.sum().toString())
var sorted_list = soundclassifier.executeScoring(short_array)
Log.w(tag, "")
for (i in 0 until 5) {
val score = sorted_list[i].value
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(tag, species_name + ", " + score.toString());
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this)
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))
// broadcastMessage(p0)
}

57
mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt Normal file
View File

@@ -0,0 +1,57 @@
package com.birdsounds.identify
import android.content.Context
import android.util.Log
import com.google.android.gms.tasks.Tasks
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import java.util.concurrent.ExecutionException
object MessageSenderFromPhone {
const val tag = "MessageSender"
private val job = Job()
private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
fun sendMessage(path: String, message: ByteArray, context: Context) {
coroutineScope.launch {
sendMessageInBackground(path, message, context)
}
}
private fun sendMessageInBackground(path: String, message: ByteArray, context: Context) {
//first get all the nodes, ie connected wearable devices.
val nodeListTask = Wearable.getNodeClient(context).connectedNodes
try {
// Block on a task and get the result synchronously (because this is on a background
// thread).
val nodes = Tasks.await(nodeListTask)
if(nodes.isEmpty()) {
Log.i(tag,"No Node found to send message")
}
//Now send the message to each device.
for (node in nodes) {
val sendMessageTask = Wearable.getMessageClient(context)
.sendMessage(node.id, path, message)
try {
// Block on a task and get the result synchronously (because this is on a background
// thread).
val result = Tasks.await(sendMessageTask)
Log.v(tag, "SendThread: message send to " + node.displayName)
} catch (exception: ExecutionException) {
Log.e(tag, "Task failed: $exception")
} catch (exception: InterruptedException) {
Log.e(tag, "Interrupt occurred: $exception")
}
}
} catch (exception: ExecutionException) {
Log.e(tag, "Task failed: $exception")
} catch (exception: InterruptedException) {
Log.e(
tag, "Interrupt occurred: $exception"
)
}
}
}

86
mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt
View File

@@ -22,6 +22,7 @@ import kotlin.concurrent.scheduleAtFixedRate
import kotlin.math.ceil
import kotlin.math.cos
import uk.me.berndporr.iirj.Butterworth
import java.nio.ShortBuffer
import kotlin.math.round
import kotlin.math.sin
@@ -31,6 +32,7 @@ class SoundClassifier(
) {
internal var mContext: Context
val TAG = "Sound Classifier"
init {
this.mContext = context.applicationContext
}
@@ -58,10 +60,10 @@ class SoundClassifier(
/** Names of the model's output classes. */
lateinit var labelList: List<String>
public lateinit var labelList: List<String>
/** Names of the model's output classes. */
lateinit var assetList: List<String>
public lateinit var assetList: List<String>
/** How many milliseconds between consecutive model inference calls. */
private var inferenceInterval = 800L
@@ -155,6 +157,7 @@ class SoundClassifier(
}
labelList = wordList.map { it.toTitleCase() }
Log.i(TAG, "Label list entries: ${labelList.size}")
} catch (e: IOException) {
Log.e(TAG, "Failed to read labels ${filename}: ${e.message}")
}
@@ -163,12 +166,16 @@ class SoundClassifier(
private fun setupInterpreter(context: Context) {
try {
val modelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.modelPath
val modelFilePath = context.getDir(
"filesdir",
Context.MODE_PRIVATE
).absolutePath + "/" + options.modelPath
Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
val modelFile = File(modelFilePath)
val tfliteBuffer: ByteBuffer = FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
val tfliteBuffer: ByteBuffer =
FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
Log.i(TAG, "Done creating TFLite buffer from $modelFilePath")
interpreter = Interpreter(tfliteBuffer, Interpreter.Options())
@@ -201,12 +208,16 @@ class SoundClassifier(
private fun setupMetaInterpreter(context: Context) {
try {
val metaModelFilePath = context.getDir("filesdir", Context.MODE_PRIVATE).absolutePath + "/"+ options.metaModelPath
val metaModelFilePath = context.getDir(
"filesdir",
Context.MODE_PRIVATE
).absolutePath + "/" + options.metaModelPath
Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath")
val metaModelFile = File(metaModelFilePath)
val tfliteBuffer: ByteBuffer = FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
val tfliteBuffer: ByteBuffer =
FileChannel.open(metaModelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
Log.i(TAG, "Done creating TFLite buffer from $metaModelFilePath")
meta_interpreter = Interpreter(tfliteBuffer, Interpreter.Options())
@@ -237,7 +248,7 @@ class SoundClassifier(
fun runMetaInterpreter(location: Location) {
val dayOfYear = LocalDate.now().dayOfYear
val week = ceil( dayOfYear*48.0/366.0) //model year has 48 weeks
val week = ceil(dayOfYear * 48.0 / 366.0) //model year has 48 weeks
lat = location.latitude.toFloat()
lon = location.longitude.toFloat()
@@ -280,6 +291,7 @@ class SoundClassifier(
}
}
private fun generateDummyAudioInput(inputBuffer: FloatBuffer) {
val twoPiTimesFreq = 2 * Math.PI.toFloat() * 1000f
for (i in 0 until modelInputLength) {
@@ -287,6 +299,7 @@ class SoundClassifier(
inputBuffer.put(i, sin(twoPiTimesFreq * x.toDouble()).toFloat())
}
}
private fun String.toTitleCase() =
splitToSequence("_")
.map { it.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.ROOT) else it.toString() } }
@@ -297,17 +310,54 @@ class SoundClassifier(
private const val TAG = "SoundClassifier"
var lat: Float = 0.0f
var lon: Float = 0.0f
/** Number of nanoseconds in a millisecond */
private const val NANOS_IN_MILLIS = 1_000_000.toDouble()
}
public fun executeScoring(
short_array: ShortArray
): List<IndexedValue<Float>> {
val highPass = 0;
val butterworth = Butterworth()
butterworth.highPass(6, 48000.0, highPass.toDouble())
val outputBuffer = FloatBuffer.allocate(modelNumClasses)
for (i in 0 until modelInputLength) {
val s = short_array[i]
if (highPass == 0) inputBuffer.put(i, s.toFloat())
else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
}
inputBuffer.rewind()
outputBuffer.rewind()
interpreter.run(inputBuffer, outputBuffer)
outputBuffer.rewind()
outputBuffer.get(predictionProbs) // Copy data to predictionProbs.
val probList = mutableListOf<Float>()
for (i in predictionProbs.indices) {
probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid
}
val outList = probList.withIndex().sortedByDescending { it -> it.value }
// Log.w(TAG, outList.toString())
// Log.i(TAG,labelList[outList[0].index].toString())
return outList
}
private fun startRecognition() {
if (modelInputLength <= 0 || modelNumClasses <= 0) {
Log.e(TAG, "Switches: Cannot start recognition because model is unavailable.")
return
}
val sharedPref = PreferenceManager.getDefaultSharedPreferences(mContext)
val highPass = sharedPref.getInt("high_pass",0)
val highPass = sharedPref.getInt("high_pass", 0)
val butterworth = Butterworth()
butterworth.highPass(6, 48000.0, highPass.toDouble())
@@ -315,7 +365,7 @@ class SoundClassifier(
var j = 0 // Indices for the circular buffer next write
Log.w(TAG, "recognitionPeriod:"+inferenceInterval)
Log.w(TAG, "recognitionPeriod:" + inferenceInterval)
recognitionTask = Timer().scheduleAtFixedRate(inferenceInterval, inferenceInterval) task@{
val outputBuffer = FloatBuffer.allocate(modelNumClasses)
val recordingBuffer = ShortArray(modelInputLength)
@@ -341,7 +391,7 @@ class SoundClassifier(
if (samplesAreAllZero && s.toInt() != 0) {
samplesAreAllZero = false
}
if (highPass==0) inputBuffer.put(i, s.toFloat())
if (highPass == 0) inputBuffer.put(i, s.toFloat())
else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
}
@@ -365,9 +415,9 @@ class SoundClassifier(
// probList.add(1 / (1 + kotlin.math.exp(-value))) //apply sigmoid
// }
// } else {
for (i in predictionProbs.indices) {
probList.add( metaPredictionProbs[i] / (1+kotlin.math.exp(-predictionProbs[i])) ) //apply sigmoid
}
for (i in predictionProbs.indices) {
probList.add(metaPredictionProbs[i] / (1 + kotlin.math.exp(-predictionProbs[i]))) //apply sigmoid
}
// }
// if (mBinding.progressHorizontal.isIndeterminate){ //if start/stop button set to "running"
@@ -375,7 +425,7 @@ class SoundClassifier(
// val max = it.maxByOrNull { entry -> entry.value }
// updateTextView(max, mBinding.text1)
// updateImage(max)
//after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax)
//after finding the maximum probability and its corresponding label (max), we filter out that entry from the list of entries before finding the second highest probability (secondMax)
// val secondMax = it.filterNot { entry -> entry == max }.maxByOrNull { entry -> entry.value }
// updateTextView(secondMax,mBinding.text2)
// }