thebears/bird_sound_identify_wearos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bump

Browse Source
This commit is contained in:
Ishan S. Patel
2025-03-08 11:07:40 -05:00
parent 06b98f107e
commit bef71f8d00
12 changed files with 208 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.idea/deploymentTargetSelector.xml generated
View File

@@ -7,10 +7,10 @@
</SelectionState>
<SelectionState runConfigName="wear">
<option name="selectionMode" value="DROPDOWN" />
<DropdownSelection timestamp="2025-03-08T01:58:56.461143900Z">
<DropdownSelection timestamp="2025-03-08T02:48:22.663246800Z">
<Target type="DEFAULT_BOOT">
<handle>
<DeviceId pluginId="Default" identifier="serial=192.168.1.195:44657;connection=82ff69d8" />
<DeviceId pluginId="Default" identifier="serial=192.168.1.195:42789;connection=6910cb2b" />
</handle>
</Target>
</DropdownSelection>

2
build.gradle.kts
View File

@@ -1,4 +1,4 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
// Top-level build file where you can add configuration options common to all sub-projects/modules.
plugins {
alias(libs.plugins.android.application) apply false
alias(libs.plugins.kotlin.android) apply false

2
gradle/libs.versions.toml
View File

@@ -21,6 +21,7 @@ media3Common = "1.4.0"
composeMaterial3 = "1.0.0-alpha23"
workRuntimeKtx = "2.9.1"
lifecycleRuntimeKtx = "2.6.1"
runtimeAndroid = "1.6.6"
#litert = "1.0.1"
[libraries]
@@ -51,6 +52,7 @@ androidx-compose-material3 = { group = "androidx.wear.compose", name = "compose-
androidx-work-runtime-ktx = { group = "androidx.work", name = "work-runtime-ktx", version.ref = "workRuntimeKtx" }
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycleRuntimeKtx" }
androidx-material3 = { group = "androidx.compose.material3", name = "material3" }
androidx-runtime-android = { group = "androidx.compose.runtime", name = "runtime-android", version.ref = "runtimeAndroid" }
#litert = { group = "com.google.ai.edge.litert", name = "litert", version.ref = "litert" }
[plugins]

68
mobile/src/main/java/com/birdsounds/identify/phone_MessageListenerService.kt
View File

@@ -4,7 +4,12 @@ package com.birdsounds.identify
import android.util.Log
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import com.theeasiestway.opus.Constants
import com.theeasiestway.opus.Opus
import decodeAACToPCM
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
import java.nio.ByteOrder
class MessageListenerService : WearableListenerService() {
@@ -12,7 +17,8 @@ class MessageListenerService : WearableListenerService() {
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
val codec_opus = Opus()
codec_opus.decoderInit(Constants.SampleRate._48000(), Constants.Channels.mono())
// MainActivity
Log.w(TAG, "Data recv: "+p0.data.size.toString() + " bytes")
val soundclassifier = MainActivity.soundClassifier
@@ -24,31 +30,49 @@ class MessageListenerService : WearableListenerService() {
}
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
var audio_bytes_og = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
var string_send: String = ""
val pcm_byte_array = decodeAACToPCM(audio_bytes)
Log.e(TAG,"Size of short array buffer: "+ pcm_byte_array.size.toString());
// ByteBuffer.wrap(audio_bytes).order(
// ByteOrder.LITTLE_ENDIAN
// ).asShortBuffer().get(short_array)
Log.e(TAG, pcm_byte_array.sum().toString())
val buffer = ByteBuffer.wrap(audio_bytes_og)
val sound_a = ByteArrayOutputStream();
val byteArrayList = mutableListOf<ByteArray>()
while (buffer.hasRemaining())
{
val num_to_read = buffer.get().toInt()
val read_this = ByteArray(num_to_read)
buffer.get(read_this)
val decoded = codec_opus.decode(read_this, Constants.FrameSize._120())
sound_a.write(decoded);
// Log.e(TAG,"Decompressed ${read_this.size} to ${decoded?.size}")
}
val audio_bytes = sound_a.toByteArray()
codec_opus.decoderRelease();
val short_array = ShortArray(audio_bytes.size/2)
// Log.e(TAG,"Size of short array buffer: "+ decoded?.size.toString());
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array)
// Log.e(TAG, pcm_byte_array.sum().toString())
Log.e(TAG, "STARTING SCORING");
// var sorted_list = soundclassifier.executeScoring(short_array)
// Log.w(TAG, "FINISHED SCORING");
// Log.w(TAG, "")
// for (i in 0 until 5) {
// val score = sorted_list[i].value
// val index = sorted_list[i].index
// val species_name = soundclassifier.labelList[index]
// Log.w(TAG, species_name + ", " + score.toString())
// string_send+= species_name
// string_send+=','
// string_send+=score.toString()
// string_send+=';'
// }
var string_send: String = ""
var sorted_list = soundclassifier.executeScoring(short_array)
Log.w(TAG, "FINISHED SCORING");
Log.w(TAG, "")
for (i in 0 until 5) {
val score = sorted_list[i].value
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(TAG, species_name + ", " + score.toString())
string_send+= species_name
string_send+=','
string_send+=score.toString()
string_send+=';'
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
}

6
wear/build.gradle.kts
View File

@@ -52,14 +52,13 @@ dependencies {
implementation("com.google.android.horologist:horologist-audio:0.6.18")
implementation("com.google.android.horologist:horologist-compose-tools:0.6.18")
implementation("com.google.android.horologist:horologist-compose-tools:0.6.18")
implementation("com.google.android.horologist:horolo+++_gist-compose-layout:0.6.18")
implementation("androidx.compose.material:material-icons-core:1.6.8")
implementation("androidx.compose.material:material-icons-extended:1.6.8")
implementation("com.google.android.horologist:horologist-compose-material:0.6.8")
implementation("com.google.android.horologist:horologist-media-ui:0.6.8")
implementation("com.google.android.horologist:horologist-media-data:0.6.8")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-play-services:1.8.1")
implementation("androidx.media3:media3-exoplayer:1.4.0")z
implementation("androidx.media3:media3-exoplayer:1.4.0")
implementation(libs.play.services.wearable)
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.ui)
@@ -74,7 +73,8 @@ dependencies {
implementation(libs.androidx.media3.common)
implementation("co.touchlab:stately-concurrent-collections:2.0.0")
implementation(libs.androidx.compose.material3)
implementation(libs.androidx.work.runtime.ktx) // androidTestImplementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.work.runtime.ktx)
implementation(libs.androidx.runtime.android) // androidTestImplementation(platform(libs.androidx.compose.bom))
// androidTestImplementation(libs.androidx.ui.test.junit4)
// debugImplementation(libs.androidx.ui.tooling)
// debugImplementation(libs.androidx.ui.test.manifest)

3
wear/src/main/java/com/birdsounds/identify/presentation/watch_ControlDashboard.kt
View File

@@ -17,8 +17,7 @@ import androidx.wear.compose.material.Text
@Composable
fun ControlDashboard(controlDashboardUiState: ControlDashboardUiState,
onMicClicked: () -> Unit,
modifier: Modifier = Modifier,
navController: NavHostController) {
modifier: Modifier = Modifier) {
Box(contentAlignment = Alignment.Center, modifier = modifier.fillMaxSize()) {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {

13
wear/src/main/java/com/birdsounds/identify/presentation/watch_MainActivity.kt
View File

@@ -105,12 +105,15 @@ fun WearApp() {
mainState.setNavController(navController);
SwipeDismissableNavHost(navController = navController, startDestination = "speaker") {
composable("species_list") {
ScreenScaffold {
SpeciesListView(context = context)
}
}
composable("speaker") {
StartRecordingScreen(
context = context,
navController = navController,
appState = mainState.appState,
isPermissionDenied = mainState.isPermissionDenied,
onMicClicked = {
scope.launch {
mainState.onMicClicked()
@@ -119,9 +122,7 @@ fun WearApp() {
)
}
composable("species_list") {
SpeciesListView(context = context)
}
}

29
wear/src/main/java/com/birdsounds/identify/presentation/watch_MessageListenerService.kt
View File

@@ -2,6 +2,7 @@ package com.birdsounds.identify.presentation
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
class MessageListenerService : WearableListenerService() {
@@ -9,19 +10,19 @@ class MessageListenerService : WearableListenerService() {
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// val t_scored = ByteBuffer.wrap(p0.data).getLong()
// var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
// var score_species_string = byte_strings.decodeToString()
// var list_strings: List<String> = score_species_string.split(';')
// list_strings.map({
// var split_str = it.split(',')
// if (split_str.size == 2) {
// var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
// if (out.score > 0.05) {
// SpeciesList.add_observation(out)
// }
// }
// })
// MessageSender.messageLog.add(t_scored)
val t_scored = ByteBuffer.wrap(p0.data).getLong()
var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
var score_species_string = byte_strings.decodeToString()
var list_strings: List<String> = score_species_string.split(';')
list_strings.map({
var split_str = it.split(',')
if (split_str.size == 2) {
var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
if (out.score > 0.005) {
SpeciesList.add_observation(out)
}
}
})
MessageSender.messageLog.add(t_scored)
}
}

111
wear/src/main/java/com/birdsounds/identify/presentation/watch_SoundRecorder.kt
View File

@@ -1,4 +1,5 @@
package com.birdsounds.identify.presentation
import com.theeasiestway.opus.Constants
import com.theeasiestway.opus.Opus
@@ -7,10 +8,13 @@ import android.content.Context
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.media.audiofx.AutomaticGainControl
import android.media.audiofx.NoiseSuppressor
import android.util.Log
import androidx.annotation.RequiresPermission
import encodePcmToAac
import kotlinx.coroutines.suspendCancellableCoroutine
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
import java.time.Instant
@@ -23,7 +27,7 @@ class SoundRecorder(
outputFileName: String
) {
private val codec = Opus();
val codec_opus = Opus();
private var state = State.IDLE
private var context = context_in
@@ -37,86 +41,97 @@ class SoundRecorder(
suspendCancellableCoroutine<Unit> { cont ->
var noiseSuppressor: NoiseSuppressor? = null
var automaticGainControl: AutomaticGainControl? = null
var chunk_index: Int = 0
val audioSource = MediaRecorder.AudioSource.DEFAULT
val sampleRateInHz = 48000
val channelConfig = AudioFormat.CHANNEL_IN_MONO
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
val buffer_size =
4 * AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat)
// Log.w(TAG, buffer_size.toString())
val bufferSizeInBytes =
sampleRateInHz * 3 * 2 // 3 second sample, 2 bytes for each sample
val frameSize = Constants.FrameSize._120();
val chunkSize = frameSize.v * 2; // Mono * 2 bytes per sample
val counts_until_reset = 3 * sampleRateInHz / frameSize.v;
val bufferSize =
AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat);
val chunk_size = 2 * sampleRateInHz / 4 // 250ms segments, 2 bytes for each sample
val num_chunks: Int = bufferSizeInBytes / chunk_size
val chunked_audio_bytes = Array(num_chunks) { ByteArray(chunk_size) }
val audio_bytes_array = ByteArray(bufferSizeInBytes)
val audioRecord = AudioRecord(
/* audioSource = */ audioSource,
/* sampleRateInHz = */ sampleRateInHz,
/* channelConfig = */ channelConfig,
/* audioFormat = */ audioFormat,
/* bufferSizeInBytes = */ buffer_size
/* bufferSizeInBytes = */ bufferSize
)
audioRecord.startRecording()
val thread = Thread {
// var sent_first: Boolean = false
val outputByteStream = ByteBuffer.allocate(1000000)
audioRecord.startRecording()
var last_tstamp: Long = Instant.now().toEpochMilli();
while (true) /**/{
var count: Int = 0;
while (true) /**/ {
if (Thread.interrupted()) {
// check for the interrupted flag, reset it, and throw exception
audioRecord.release();
Log.w(TAG, "Finished thread")
break
}
chunk_index = chunk_index.mod(num_chunks)
if (chunk_index == 0) {
codec.encoderInit(48000, 1, Constants.Application.audio());
if (count == 0) {
codec_opus.encoderInit(
Constants.SampleRate._48000(),
Constants.Channels.mono(),
Constants.Application.audio()
);
outputByteStream.clear();
}
val out = audioRecord.read(
/* audioData = */ chunked_audio_bytes[chunk_index],
/* offsetInBytes = */ 0,
/* sizeInBytes = */ chunk_size,
/* readMode = */ AudioRecord.READ_BLOCKING
)
val frame = ByteArray(chunkSize)
var offset = 0
var remained = frame.size
while (remained > 0) {
val read = audioRecord.read(frame, offset, remained)
offset += read
remained -= read
}
val encoded: ByteArray? =
codec_opus.encode(bytes = frame, frameSize = frameSize);
if (encoded != null) {
outputByteStream.put(encoded.size.toByte());
outputByteStream.put(encoded);
}
chunk_index += 1
count += 1;
if (count == counts_until_reset) {
Log.d(TAG, "At count reset at ${count}");
codec_opus.encoderRelease();
count = 0;
val writtenBytes = outputByteStream.position()
Log.e(TAG,"Wrote ${writtenBytes} bytes!")
val duplicate = outputByteStream.duplicate()
duplicate.flip() // Prepare for reading
// outputByteStream.flip()
val bytes = ByteArray(writtenBytes)
duplicate.get(bytes)
// val result = ByteArray(outputByteStream.remaining());
var tstamp: Long = Instant.now().toEpochMilli()
val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
var byte_send: ByteArray = ByteArray(0)
var strr: String = ""
for (i in 0..(num_chunks - 1)) {
var c_index = i + chunk_index
c_index = c_index.mod(num_chunks)
strr += c_index.toString()
strr += ' '
byte_send += chunked_audio_bytes[c_index]
}
// do_send_message = false;
// num_chunked_since_last_send = 0
// ignore_warmup = false;
MessageSender.messageLog.clear()
val compressed = encodePcmToAac(byte_send)
Log.i(TAG,"Size pre-compression "+byte_send.size.toString())
Log.i(TAG,"Size post-compression "+compressed.size.toString())
MessageSender.sendMessage("/audio", compressed, context)
last_tstamp = tstamp
var byte_send: ByteArray = tstamp_bytes + bytes;
Log.e(TAG, "Sending message of 3s with size ${byte_send.size} + ${byte_send[0].toString()}")
MessageSender.sendMessage("/audio", byte_send, context)
}
}
}
thread.start()
// thread.join();

43
wear/src/main/java/com/birdsounds/identify/presentation/watch_SpeciesList.kt
View File

@@ -2,6 +2,7 @@ package com.birdsounds.identify.presentation
import android.util.Log
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.snapshots.SnapshotStateList
import java.time.Instant
@@ -13,16 +14,18 @@ class AScore(
) {
var split_stuff = _species.split("_")
val species = split_stuff[0]
val score = _score
val common_name = split_stuff[1]
val split_stuff: List<String> = _species.split("_");
val species = split_stuff[0];
val score = _score;
// var common_name = split_stuff[1];
val common_name = if (split_stuff.size > 1) split_stuff[1] else "";
val timestamp = _timestamp
fun age(): Long {
var tstamp: Long = Instant.now().toEpochMilli()
return (tstamp - timestamp)
}
override fun toString(): String {
var tstamp: Long = Instant.now().toEpochMilli()
return common_name + "," + species + "," + score.toString() + ", " + (age() / 1000.0).toString() + "s ago"
@@ -31,20 +34,20 @@ class AScore(
object SpeciesList {
var internal_list = mutableListOf<AScore>()
var internal_list = mutableListOf<MutableState<AScore>>()
var do_add_observation = false
var _list_on_ui: SnapshotStateList<MutableState<String>>? = null
fun setSpeciecsShow_list(list_in: SnapshotStateList<MutableState<String>>) {
var _list_on_ui: SnapshotStateList<MutableState<AScore>>? = null
fun setSpeciecsShow_list(list_in: SnapshotStateList<MutableState<AScore>>) {
_list_on_ui = list_in;
}
fun add_observation(species_in: AScore) {
Log.w(TAG,"In add obsergation")
Log.w(TAG, "In add obervation")
do_add_observation = false
var idx = 0
var idx_replace = -1
for (i in internal_list) {
if (i.species == species_in.species) {
if (i.value.species == species_in.species) {
do_add_observation = false
idx_replace = idx
}
@@ -52,7 +55,8 @@ object SpeciesList {
}
if (idx_replace >= 0) {
Log.w(TAG, "Replacing")
internal_list[idx_replace] = species_in // _list_on_ui?.removeAt(idx_replace)
internal_list[idx_replace].value =
species_in // _list_on_ui?.removeAt(idx_replace)
// _list_on_ui?.add(species_in);
// if (_list_on_ui != null) {
// _list_on_ui?.removeAt(idx_replace)
@@ -60,13 +64,26 @@ object SpeciesList {
// }
} else {
internal_list.add(species_in)
internal_list.add(mutableStateOf(species_in))
}
internal_list = internal_list.sortedBy({ (it.age()) }).toMutableList()
internal_list = internal_list.sortedBy({ (it.value.age()) }).toMutableList()
while (_list_on_ui!!.size < internal_list.size) {
_list_on_ui!!.add(mutableStateOf(AScore("", 0F, 0L)))
Log.w(TAG, "Adding stuff to UI list");
}
Log.w(TAG, "Internal list " + internal_list.toString())
for ((index, value) in internal_list.withIndex()) {
_list_on_ui?.get(index)?.value = value.common_name;
Log.w(TAG, "Adding ${index}:${value} to list ${_list_on_ui!!.size}")
if (_list_on_ui!!.size > index)
{
Log.w(TAG,"Updating ${value}")
_list_on_ui!![index].value = value.value
}
}
Log.w(TAG, internal_list.size.toString())

63
wear/src/main/java/com/birdsounds/identify/presentation/watch_SpeciesListView.kt
View File

@@ -1,48 +1,59 @@
package com.birdsounds.identify.presentation
import android.content.Context
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.runtime.Composable
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.snapshots.SnapshotStateList
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.wear.compose.foundation.lazy.items
import androidx.wear.compose.material.Text
import com.google.android.horologist.annotations.ExperimentalHorologistApi
import com.google.android.horologist.compose.layout.ScalingLazyColumn
import com.google.android.horologist.compose.layout.ScalingLazyColumnDefaults
import com.google.android.horologist.compose.layout.ScreenScaffold
import com.google.android.horologist.compose.layout.rememberResponsiveColumnState
@OptIn(ExperimentalHorologistApi::class)
@OptIn(ExperimentalHorologistApi::class)
@Composable
fun SpeciesListView(context: Context,
) {
val text: MutableState<String> = mutableStateOf("text")
//text.toString()
val species_list_show = mutableStateListOf<MutableState<String>>()
for (i in 1..10)
{
val hi = mutableStateOf("hi")
species_list_show.add(hi);
fun SpeciesListView(
context: Context,
) {
val species_list_show = mutableStateListOf<MutableState<AScore>>()
for (i in 1..10) {
species_list_show.add(mutableStateOf(AScore(i.toString()+"_"+i.toString(), 0.00F, 0L)));
}
SpeciesList.setSpeciecsShow_list(species_list_show)
val species_show: SnapshotStateList<MutableState<String>> = remember { species_list_show }
val columnState =
rememberResponsiveColumnState(contentPadding = ScalingLazyColumnDefaults.padding(first = ScalingLazyColumnDefaults.ItemType.Text,
last = ScalingLazyColumnDefaults.ItemType.Chip))
ScreenScaffold(scrollState = columnState) {
ScalingLazyColumn(columnState = columnState, modifier = Modifier.fillMaxSize()) {
items(species_show) { aSpec -> Text(text = aSpec.value)
} // Dynamically display the chips
}
val species_show: SnapshotStateList<MutableState<AScore>> = remember { species_list_show }
Column(
horizontalAlignment = Alignment.CenterHorizontally,
modifier = Modifier
.verticalScroll(rememberScrollState())
.fillMaxWidth()
) {
species_show.forEach { aSpec -> Text(text = aSpec.value.common_name) }
}
}
}
// val columnState =
// rememberResponsiveColumnState(contentPadding = ScalingLazyColumnDefaults.padding(first = ScalingLazyColumnDefaults.ItemType.Text,
// last = ScalingLazyColumnDefaults.ItemType.Chip))
// ScreenScaffold(scrollState = columnState) {
// ScalingLazyColumn(columnState = columnState, modifier = Modifier.fillMaxSize()) {
// items(species_show) { aSpec -> Text(text = aSpec.value)
// } // Dynamically display the chips
//
// }
// }

6
wear/src/main/java/com/birdsounds/identify/presentation/watch_StartRecordingScreen.kt
View File

@@ -9,10 +9,7 @@ import com.google.android.horologist.compose.layout.ScreenScaffold
@Composable
fun StartRecordingScreen(
context: Context,
appState: AppState,
navController: NavHostController,
isPermissionDenied: Boolean,
onMicClicked: () -> Unit
) {
ScreenScaffold {
@@ -21,8 +18,7 @@ fun StartRecordingScreen(
)
ControlDashboard(
controlDashboardUiState = controlDashboardUiState,
onMicClicked = onMicClicked,
navController = navController
onMicClicked = onMicClicked
)
}