thebears/bird_sound_identify_wearos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump˚

Browse Source
This commit is contained in:
isp
2024-09-13 22:53:43 -06:00
parent 3f701fefd3
commit c5824c9439
13 changed files with 182 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/codeStyles/codeStyleConfig.xml generated
View File

@@ -1,6 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>

14
mobile/src/main/java/com/birdsounds/identify/MainActivity.kt
View File

@@ -15,6 +15,8 @@ import com.google.android.gms.wearable.Wearable
class MainActivity : AppCompatActivity() {
// private lateinit var soundClassifier: SoundClassifier
val REQUEST_PERMISSIONS = 1337
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
@@ -28,20 +30,22 @@ class MainActivity : AppCompatActivity() {
}
}
)
Downloader.downloadModels(this);
requestPermissions();
Downloader.downloadModels(this)
requestPermissions()
soundClassifier = SoundClassifier(this, SoundClassifier.Options())
Location.requestLocation(this, soundClassifier)
ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets ->
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom)
insets
}
}
companion object {
const val REQUEST_PERMISSIONS = 1337
@SuppressLint("StaticFieldLeak")
lateinit var soundClassifier: SoundClassifier
var soundClassifier: SoundClassifier? = null
// fun getSoundClassifier(): SoundClassifier? {
// return soundClassifier
// }
}
private fun requestPermissions() {

14
mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt
View File

@@ -18,16 +18,18 @@ class MessageListenerService : WearableListenerService() {
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// MainActivity
val soundclassifier = MainActivity.soundClassifier
if (soundclassifier == null) {
Log.w(tag, "Have invalid sound classifier")
return
} else {
Log.w(tag, "Have valid classifier")
}
val short_array = ShortArray(48000 * 3);
val short_array = ShortArray(48000 * 3)
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
var string_send: String = ""
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
@@ -39,9 +41,13 @@ class MessageListenerService : WearableListenerService() {
val score = sorted_list[i].value
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(tag, species_name + ", " + score.toString());
Log.w(tag, species_name + ", " + score.toString())
string_send+= species_name
string_send+=','
string_send+=score.toString()
string_send+=';'
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes, this)
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))

12
mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt
View File

@@ -5,6 +5,7 @@ import android.location.Location
import android.os.SystemClock
import android.preference.PreferenceManager
import android.util.Log
import androidx.annotation.Nullable
import org.tensorflow.lite.Interpreter
import java.io.BufferedReader
import java.io.File
@@ -26,6 +27,7 @@ import java.nio.ShortBuffer
import kotlin.math.round
import kotlin.math.sin
class SoundClassifier(
context: Context,
private val options: Options = Options()
@@ -60,10 +62,10 @@ class SoundClassifier(
/** Names of the model's output classes. */
public lateinit var labelList: List<String>
lateinit var labelList: List<String>
/** Names of the model's output classes. */
public lateinit var assetList: List<String>
lateinit var assetList: List<String>
/** How many milliseconds between consecutive model inference calls. */
private var inferenceInterval = 800L
@@ -315,10 +317,10 @@ class SoundClassifier(
private const val NANOS_IN_MILLIS = 1_000_000.toDouble()
}
public fun executeScoring(
fun executeScoring(
short_array: ShortArray
): List<IndexedValue<Float>> {
val highPass = 0;
val highPass = 0
val butterworth = Butterworth()
butterworth.highPass(6, 48000.0, highPass.toDouble())
@@ -372,7 +374,7 @@ class SoundClassifier(
// Load new audio samples
// val sampleCounts = loadAudio(recordingBuffer)
val sampleCounts = 0;
val sampleCounts = 0
if (sampleCounts == 0) {
return@task
}

2
wear/src/main/AndroidManifest.xml
View File

@@ -37,7 +37,7 @@
<service
android:name=".MessageListenerService"
android:name=".presentation.MessageListenerService"
android:enabled="true"
android:exported="true" >
<intent-filter>

35
wear/src/main/java/com/birdsounds/identify/MessageListenerService.kt
View File

@@ -1,35 +0,0 @@
package com.birdsounds.identify
import android.app.Service
import android.content.Intent
import android.util.Half.abs
import android.util.Log
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import com.birdsounds.identify.presentation.MessageSender
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.ShortBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
val t_scored = ByteBuffer.wrap(p0.data).getLong()
MessageSender.messageLog.add(t_scored)
Log.w("MSG_RECV", t_scored.toString())
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))
// broadcastMessage(p0)
}
}

45
wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt
View File

@@ -16,13 +16,11 @@ import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.compose.setContent
import androidx.activity.result.contract.ActivityResultContracts.RequestPermission
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.tooling.preview.Preview
import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen
import androidx.lifecycle.DefaultLifecycleObserver
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.viewmodel.compose.viewModel
import androidx.wear.compose.navigation.SwipeDismissableNavHost
import androidx.wear.compose.navigation.composable
@@ -30,11 +28,8 @@ import androidx.wear.compose.navigation.rememberSwipeDismissableNavController
import com.birdsounds.identify.presentation.theme.IdentifyTheme
import com.google.android.horologist.annotations.ExperimentalHorologistApi
import com.google.android.horologist.audio.ui.VolumeViewModel
import com.google.android.horologist.compose.layout.AppScaffold
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.*
val flow_stream = MutableStateFlow<String>("")
@@ -63,17 +58,11 @@ class MainActivity : ComponentActivity() {
@OptIn(ExperimentalHorologistApi::class)
@Composable
@Preview
fun WearApp() {
IdentifyTheme {
lateinit var requestPermissionLauncher: ManagedActivityResultLauncher<String, Boolean>
// val job = launch {
// flow_stream.collect {
// print("$it ")
// }
// }
val context = LocalContext.current
val activity = context.findActivity()
val scope = rememberCoroutineScope()
@@ -99,20 +88,20 @@ fun WearApp() {
val lifecycleOwner = androidx.lifecycle.compose.LocalLifecycleOwner.current
AppScaffold {
DisposableEffect(mainState, scope, lifecycleOwner) {
val lifecycleObserver = object : DefaultLifecycleObserver {
override fun onStop(owner: LifecycleOwner) {
super.onStop(owner)
}
}
lifecycleOwner.lifecycle.addObserver(lifecycleObserver)
onDispose {
lifecycleOwner.lifecycle.removeObserver(lifecycleObserver)
}
}
SwipeDismissableNavHost(navController = navController, startDestination = "speaker") {
// AppScaffold {
// DisposableEffect(mainState, scope, lifecycleOwner) {
// val lifecycleObserver = object : DefaultLifecycleObserver {
// override fun onStop(owner: LifecycleOwner) {
// super.onStop(owner)
// }
// }
// lifecycleOwner.lifecycle.addObserver(lifecycleObserver)
//
// onDispose {
// lifecycleOwner.lifecycle.removeObserver(lifecycleObserver)
// }
// }
var sdnv = SwipeDismissableNavHost(navController = navController, startDestination = "speaker") {
composable("speaker") {
StartRecordingScreen(
context = context,
@@ -125,7 +114,7 @@ fun WearApp() {
},
)
}
}
// }
}
}
}

8
wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt
View File

@@ -46,7 +46,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
when (appState) {
is AppState.Ready -> when {
(ContextCompat.checkSelfPermission(activity, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) -> {
Log.e(TAG, "Permissions granted, continuing to record");
Log.e(TAG, "Permissions granted, continuing to record")
appState = AppState.Recording
@@ -54,7 +54,7 @@ class MainState(private val activity: Activity, private val requestPermission: (
}
else -> {
Log.e(TAG, "Requesting permissions");
Log.e(TAG, "Requesting permissions")
requestPermission()
}
}
@@ -100,7 +100,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
coroutineScope { // Kick off a parallel job to
Log.e(TAG, "Start recording"); //
Log.e(TAG, "Start recording") //
val recordingJob = launch { soundRecorder.record() }
// val recordingJob = launch { soundRecorder.record() }
// SoundRecorder.record();
@@ -142,7 +142,7 @@ private suspend fun record(activity: (Activity),soundRecorder: SoundRecorder,
}
object channelCallback : ChannelClient.ChannelCallback() {
object channelCallback : ChannelCallback() {
override fun onChannelOpened(channel: ChannelClient.Channel) {
super.onChannelOpened(channel)
Log.e(TAG,"Opened channel")}

28
wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt Normal file
View File

@@ -0,0 +1,28 @@
package com.birdsounds.identify.presentation
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
val t_scored = ByteBuffer.wrap(p0.data).getLong()
var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
var score_species_string = byte_strings.decodeToString()
var list_strings: List<String> = score_species_string.split(';')
list_strings.map({
var split_str = it.split(',')
if (split_str.size == 2) {
var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
if (out.score > 0.05) {
SpeciesList.add_observation(out)
}
}
})
MessageSender.messageLog.add(t_scored)
}
}

5
wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt
View File

@@ -3,10 +3,8 @@ package com.birdsounds.identify.presentation
import android.content.Context
import android.util.Log
import co.touchlab.stately.collections.ConcurrentMutableCollection
import co.touchlab.stately.collections.ConcurrentMutableList
import co.touchlab.stately.collections.ConcurrentMutableSet
import com.google.android.gms.tasks.Tasks
import com.google.android.gms.tasks.Tasksx
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -14,6 +12,7 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import java.util.concurrent.ExecutionException
object MessageSender {
const val tag = "MessageSender"
private val job = Job()

54
wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt
View File

@@ -6,10 +6,8 @@ import android.content.Context
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.os.Message
import android.util.Log
import androidx.annotation.RequiresPermission
import co.touchlab.stately.collections.ConcurrentMutableCollection
import kotlinx.coroutines.suspendCancellableCoroutine
import java.nio.ByteBuffer
import java.time.Instant
@@ -46,7 +44,7 @@ class SoundRecorder(
// Log.w(TAG, buffer_size.toString())
val bufferSizeInBytes =
sampleRateInHz * 3 * 2; // 3 second sample, 2 bytes for each sample
sampleRateInHz * 3 * 2 // 3 second sample, 2 bytes for each sample
val chunk_size = 2 * sampleRateInHz / 4 // 250ms segments, 2 bytes for each sample
val num_chunks: Int = bufferSizeInBytes / chunk_size
@@ -62,25 +60,44 @@ class SoundRecorder(
)
audioRecord.startRecording()
val thread = Thread {
var num_interactions = 0
// var sent_first: Boolean = false
var ignore_warmup: Boolean = true
var num_chunked_since_last_send = 0
var last_tstamp: Long = Instant.now().toEpochMilli()
var do_send_message: Boolean = false
while (true) {
if (Thread.interrupted()) {
// check for the interrupted flag, reset it, and throw exception
Log.w(TAG, "Finished thread");
break;
Log.w(TAG, "Finished thread")
break
}
chunk_index = chunk_index.mod(num_chunks)
val out = audioRecord.read(
/* audioData = */ chunked_audio_bytes[chunk_index],
/* offsetInBytes = */ 0,
/* sizeInBytes = */ chunk_size,
/* readMode = */ android.media.AudioRecord.READ_BLOCKING
/* readMode = */ AudioRecord.READ_BLOCKING
)
num_chunked_since_last_send += 1
chunk_index += 1;
if ((last_tstamp in MessageSender.messageLog) || (num_interactions == 0)) {
do_send_message = false
if (num_chunked_since_last_send >= num_chunks) {
do_send_message = true
Log.w("MSG","sending message because full 3s have passed")
} else if ((last_tstamp in MessageSender.messageLog) && (num_chunked_since_last_send>4)) {
do_send_message = true
Log.w("MSG","Send message because the phone has finished")
} else if ((ignore_warmup) && (num_chunked_since_last_send > 2)) {
do_send_message = true
Log.w("MSG","Sent message because ignoring warmup")
}
chunk_index += 1
if ((do_send_message)) {
var tstamp: Long = Instant.now().toEpochMilli()
val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
@@ -89,24 +106,25 @@ class SoundRecorder(
for (i in 0..(num_chunks - 1)) {
var c_index = i + chunk_index
c_index = c_index.mod(num_chunks)
strr += c_index.toString();
strr += c_index.toString()
strr += ' '
byte_send += chunked_audio_bytes[c_index]
}
// Log.w(TAG, strr)
// Log.w("MSG_SENT",byte_send.sum().toString()+",. "+ tstamp.toString())
// do_send_message = false;
num_chunked_since_last_send = 0
MessageSender.messageLog.clear()
MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context)
last_tstamp = tstamp;
}
last_tstamp = tstamp
}
};
thread.start();
}
}
thread.start()
// thread.join();
cont.invokeOnCancellation {
thread.interrupt();
audioRecord.stop();
thread.interrupt()
audioRecord.stop()
audioRecord.release()
state = State.IDLE
}

62
wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt Normal file
View File

@@ -0,0 +1,62 @@
package com.birdsounds.identify.presentation
import android.util.Log
import java.time.Instant
class AScore(
_species: String,
_score: Float,
_timestamp: Long
) {
var split_stuff = _species.split("_")
val species = split_stuff[0]
val score = _score
val common_name = split_stuff[1]
val timestamp = _timestamp
fun age(): Long {
var tstamp: Long = Instant.now().toEpochMilli()
return (tstamp - timestamp)
}
override fun toString(): String {
var tstamp: Long = Instant.now().toEpochMilli()
return common_name + "," + species + "," + score.toString() + ", " + (age() / 1000.0).toString() + "s ago"
}
}
object SpeciesList {
var internal_list = mutableListOf<AScore>()
var do_add_observation = false
fun add_observation(species_in: AScore) {
do_add_observation = false
var idx = 0
var idx_replace = -1
for (i in internal_list)
{
if (i.species == species_in.species)
{
do_add_observation = false
idx_replace = idx
}
idx+=1
}
if (idx_replace >= 0)
{
Log.w(TAG, "Replacing")
internal_list[idx_replace] = species_in
} else {
internal_list.add(species_in)
}
// val list_reranked = internal_list.withIndex().sortedBy { it -> it.value.age() }
internal_list = internal_list.sortedBy({ (it.age()) }).toMutableList()
Log.w(TAG, internal_list.toString())
// internal_list.add(species_in)
}
}

6
wear/src/main/res/layout/layout.xml Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent">
</LinearLayout>