Restore latest local work after Seafile git corruption repair

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TvDdasWXdhrvQriBCPfFn1
This commit is contained in:
2026-08-30 22:19:01 -04:00
parent e73d52d221
commit a70b507c18
29 changed files with 1089 additions and 1163 deletions
@@ -1,73 +0,0 @@
plugins {
alias(libs.plugins.android.application)
alias(libs.plugins.kotlin.android)
alias(libs.plugins.kotlin.compose)
}
android {
namespace = "com.birdsounds.identify"
compileSdk = 34
defaultConfig {
applicationId = "com.birdsounds.identify"
minSdk = 34
targetSdk = 34
versionCode = 1
versionName = "1.0"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
}
buildTypes {
release {
isMinifyEnabled = false
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro"
)
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
}
kotlinOptions {
jvmTarget = "1.8"
}
buildFeatures {
compose = true
}
}
dependencies {
implementation(libs.androidx.core.ktx)
implementation(libs.androidx.appcompat)
implementation(libs.play.services.wearable)
implementation(libs.material)
implementation(libs.androidx.activity)
implementation(libs.androidx.constraintlayout)
implementation(libs.androidx.work.runtime.ktx)
implementation(libs.androidx.lifecycle.runtime.ktx)
implementation(libs.androidx.activity.compose)
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.ui)
implementation("uk.me.berndporr:iirj:1.7")
implementation(libs.androidx.ui.graphics)
implementation(libs.androidx.ui.tooling.preview)
implementation(libs.androidx.material3)
implementation(libs.litert)
testImplementation(libs.junit)
androidTestImplementation(libs.androidx.junit)
androidTestImplementation(libs.androidx.espresso.core)
androidTestImplementation(platform(libs.androidx.compose.bom))
androidTestImplementation(libs.androidx.ui.test.junit4)
debugImplementation(libs.androidx.ui.tooling)
debugImplementation(libs.androidx.ui.test.manifest)
implementation("org.tensorflow:tensorflow-lite-gpu:2.12.0")
implementation("org.tensorflow:tensorflow-lite-support:0.4.4")
implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.4")
implementation("org.tensorflow:tensorflow-lite-task-text:0.4.4")
implementation("com.google.android.gms:play-services-tflite-gpu:16.2.0")
implementation("com.google.android.gms:play-services-tflite-java:16.0.0-beta01")
wearApp(project(":wear"))
}
+10 -2
View File
@@ -51,13 +51,19 @@ dependencies {
implementation(libs.androidx.activity)
implementation(libs.androidx.constraintlayout)
implementation(libs.androidx.work.runtime.ktx)
// implementation("org.tensorflow:tensorflow-lite-task-vision")
// implementation("org.tensorflow:tensorflow-lite-gpu-delegate-plugin")
implementation(libs.androidx.lifecycle.runtime.ktx)
implementation(libs.androidx.activity.compose)
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.ui)
implementation("io.github.nailik:androidresampler:0.1")
// implementation("org.tensorflow:tensorflow-lite:1.12.0")
implementation("org.tensorflow:tensorflow-lite:2.16.1")
implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.4")
implementation("androidx.datastore:datastore-preferences:1.1.3")
implementation("org.tensorflow:tensorflow-lite:2.6.0")
implementation("androidx.datastore:datastore-preferences:1.1.3")
// implementation("org.tensorflow:tensorflow-lite:2.6.0")
// implementation("com.google.android.gms:play-services-tflite:20.0.0")
implementation("uk.me.berndporr:iirj:1.7")
implementation(libs.androidx.ui.graphics)
@@ -65,6 +71,8 @@ dependencies {
implementation(libs.androidx.material3)
implementation(libs.androidx.datastore.core.android)
implementation(libs.core.ktx)
// implementation(libs.litert.support.api)
// implementation(libs.litert)
// implementation(libs.litert)
testImplementation(libs.junit)
androidTestImplementation(libs.androidx.junit)
Binary file not shown.
@@ -42,11 +42,13 @@ class Downloader(mainActivity: MainActivity) {
}
}
fun prepareModelFiles()
{
copyAssetToFolder(settings.pkg_model_file, settings.local_model_file);
copyAssetToFolder(settings.pkg_meta_model_file, settings.local_meta_model_file);
fun prepareModelFiles() {
for (files_to_copy in Settings.all_files) {
copyAssetToFolder(
files_to_copy["pkg"].toString(),
files_to_copy["file"].toString()
)
}
}
}
@@ -31,7 +31,11 @@ object Location {
locationListenerGPS = null
}
fun requestLocation(context: Context, soundClassifier: SoundClassifier) {
fun requestLocation(
context: Context,
soundClassifier: SoundClassifier,
soundClassifierSpectrogramBased: SoundClassifierSpectrogramBased
) {
if (ActivityCompat.checkSelfPermission(
context,
Manifest.permission.ACCESS_COARSE_LOCATION
@@ -44,18 +48,28 @@ object Location {
if (locationListenerGPS == null) locationListenerGPS = object : LocationListener {
@SuppressLint("SetTextI18n")
override fun onLocationChanged(location: Location) {
Log.w(TAG, "Got location changed");
while (!soundClassifier.is_model_ready()) {
Thread.sleep(50);
}
Log.w(TAG, "Sound classifier is ready");
soundClassifier.runMetaInterpreter(location)
Log.w(TAG, "Got location changed");
while (!soundClassifier.is_model_ready()) {
Thread.sleep(50);
}
Log.w(TAG, "Sound classifier is ready");
soundClassifier.runMetaInterpreter(location)
Log.w(TAG, "Got location changed");
while (!soundClassifierSpectrogramBased.is_model_ready()) {
Thread.sleep(50);
}
Log.w(TAG, "Sound classifier is ready");
soundClassifierSpectrogramBased.runMetaInterpreter(location)
val activity = context as? Activity;
activity?.let {
val text_species: TextView = it.findViewById(R.id.local_species)
text_species.text = local_species;
text_species.text = local_species_spectrogram;
val loc_lon: TextView = it.findViewById(R.id.location_long)
loc_lon.text =
@@ -71,12 +85,7 @@ object Location {
geocoder.getFromLocation(location.latitude, location.longitude, 1)
if (addresses?.isNotEmpty() == true) {
val address: Address = addresses[0]
// loc_string.text = address.locality.toString() + ", " + address.adminArea.toString() + " " + address.countryName.toString()
loc_string.text = address.getAddressLine(0).toString()
// Log.w(TAG, address.toString())
}
}
@@ -4,9 +4,10 @@ import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.content.pm.PackageManager
import android.graphics.Bitmap
import android.os.Bundle
import android.preference.PreferenceManager
import android.util.Log
import android.widget.ImageView
import android.widget.SeekBar
import android.widget.TextView
import androidx.activity.enableEdgeToEdge
@@ -22,11 +23,13 @@ import androidx.datastore.preferences.preferencesDataStore
import androidx.lifecycle.lifecycleScope
import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
import io.github.nailik.androidresampler.Resampler
import io.github.nailik.androidresampler.ResamplerConfiguration
import io.github.nailik.androidresampler.data.ResamplerChannel
import io.github.nailik.androidresampler.data.ResamplerQuality
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
@@ -36,8 +39,6 @@ private var updateJob: Job? = null
private var updateCounter = 0
val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = "settings")
val Any.TAG: String
get() {
val tag = javaClass.simpleName
@@ -49,30 +50,64 @@ class SynchronousDataStore(private val context: Context) {
// Define keys
companion object {
val THRESHOLD_KEY = intPreferencesKey("user_age")
val THRESHOLD_SPECTROGRAM_KEY = intPreferencesKey("THRESHOLD_SPECTROGRAM")
val THRESHOLD_WAVEFORM_KEY = intPreferencesKey("THRESHOLD_WAVEFORM")
val THRESHOLD_BIRDHEARD_KEY = intPreferencesKey("THRESHOLD_BIRD_HEARD")
}
// Synchronous write operations
fun saveThreshold(value: Int) {
fun saveThresholdSpectrogram(value: Int) {
runBlocking {
context.dataStore.edit { preferences ->
preferences[THRESHOLD_KEY] = value;
preferences[THRESHOLD_SPECTROGRAM_KEY] = value;
}
}
}
fun getThreshold(): Int {
fun getThresholdSpectrogram(): Int {
return runBlocking {
context.dataStore.data.first()[THRESHOLD_KEY] ?: 50
}
return runBlocking {
context.dataStore.data.first()[THRESHOLD_SPECTROGRAM_KEY] ?: 50
}
}
// Synchronous write operations
fun saveThresholdWaveform(value: Int) {
runBlocking {
context.dataStore.edit { preferences ->
preferences[THRESHOLD_WAVEFORM_KEY] = value;
}
}
}
fun getThresholdWaveform(): Int {
return runBlocking {
context.dataStore.data.first()[THRESHOLD_WAVEFORM_KEY] ?: 50
}
}
// Synchronous write operations
fun saveBirdHeardThreshold(value: Int) {
runBlocking {
context.dataStore.edit { preferences ->
preferences[THRESHOLD_BIRDHEARD_KEY] = value;
}
}
}
fun getBirdHeardThreshold(): Int {
return runBlocking {
context.dataStore.data.first()[THRESHOLD_BIRDHEARD_KEY] ?: 50
}
}
}
class MainActivity : AppCompatActivity() {
// private lateinit var soundClassifier: SoundClassifier
val REQUEST_PERMISSIONS = 1337
private lateinit var dataStore: SynchronousDataStore
@@ -88,15 +123,16 @@ class MainActivity : AppCompatActivity() {
super.onChannelOpened(channel)
Log.d(TAG, "onChannelOpened")
}
}
)
})
Downloader(this).prepareModelFiles();
Log.w(TAG, "Finished setting up downloader")
requestPermissions()
soundClassifier = SoundClassifier(this, SoundClassifier.Options())
Log.w(TAG, "Starting sound classifier")
Location.requestLocation(this, soundClassifier!!)
Log.w(TAG, "Starting location requester")
soundClassifierSpectrogramBased = SoundClassifierSpectrogramBased(this);
Location.requestLocation(this, soundClassifier!!, soundClassifierSpectrogramBased!!);
ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets ->
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom)
@@ -104,26 +140,79 @@ class MainActivity : AppCompatActivity() {
}
val thresholdText = findViewById<TextView>(R.id.threshold_value_text)
val seekBar = findViewById<SeekBar>(R.id.threshold_set_scale_bar)
last_message_delay = findViewById<TextView>(R.id.last_message_delay)
dataStore = SynchronousDataStore(applicationContext)
dataStore = SynchronousDataStore(applicationContext)
val configuration = ResamplerConfiguration(
quality = ResamplerQuality.BEST,
inputChannel = ResamplerChannel.MONO,
inputSampleRate = 48000,
outputChannel = ResamplerChannel.MONO,
outputSampleRate = 16000
)
resampler = Resampler(configuration);
Settings.threshold = dataStore.getThreshold();
seekBar.progress = Settings.threshold
thresholdText.text = String.format("%.2f", Settings.threshold/100f)
seekBar.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener {
val thresholdText_birdsinging = findViewById<TextView>(R.id.threshold_value_text_birdheard)
val seekBar_bird_singing = findViewById<SeekBar>(R.id.threshold_set_scale_bar_birdheard);
Settings.threshold_bird_singing = dataStore.getBirdHeardThreshold();
seekBar_bird_singing.progress = Settings.threshold_bird_singing
thresholdText_birdsinging.text =
String.format("%.2f", Settings.threshold_bird_singing / 100f)
seekBar_bird_singing.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener {
@SuppressLint("DefaultLocale")
override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {
var actualValue = progress
dataStore.saveThreshold(actualValue);
Settings.threshold = actualValue;
thresholdText.text = String.format("%.2f", actualValue/100f)
dataStore.saveBirdHeardThreshold(actualValue);
Settings.threshold_bird_singing = actualValue;
thresholdText_birdsinging.text = String.format("%.2f", actualValue / 100f)
}
override fun onStartTrackingTouch(seekBar: SeekBar) {}
override fun onStopTrackingTouch(seekBar: SeekBar) {}
})
val thresholdText_waveform = findViewById<TextView>(R.id.threshold_value_text_waveform)
val seekBar_Waveform = findViewById<SeekBar>(R.id.threshold_set_scale_bar_waveform);
Settings.threshold_waveform = dataStore.getThresholdWaveform();
seekBar_Waveform.progress = Settings.threshold_waveform
thresholdText_waveform.text = String.format("%.2f", Settings.threshold_waveform / 100f)
seekBar_Waveform.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener {
@SuppressLint("DefaultLocale")
override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {
var actualValue = progress
dataStore.saveThresholdWaveform(actualValue);
Settings.threshold_waveform = actualValue;
thresholdText_waveform.text = String.format("%.2f", actualValue / 100f)
}
override fun onStartTrackingTouch(seekBar: SeekBar) {}
override fun onStopTrackingTouch(seekBar: SeekBar) {}
})
val thresholdText_spectrogram =
findViewById<TextView>(R.id.threshold_value_text_spectrogram)
val seekBar_Spectrogram = findViewById<SeekBar>(R.id.threshold_set_scale_bar_spectrogram);
Settings.threshold_spectrogram = dataStore.getThresholdSpectrogram();
seekBar_Spectrogram.progress = Settings.threshold_spectrogram
thresholdText_spectrogram.text =
String.format("%.2f", Settings.threshold_spectrogram / 100f)
seekBar_Spectrogram.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener {
@SuppressLint("DefaultLocale")
override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {
var actualValue = progress
dataStore.saveThresholdSpectrogram(actualValue);
Settings.threshold_spectrogram = actualValue;
thresholdText_spectrogram.text = String.format("%.2f", actualValue / 100f)
}
override fun onStartTrackingTouch(seekBar: SeekBar) {}
@@ -132,11 +221,21 @@ class MainActivity : AppCompatActivity() {
})
}
fun set_image_view(bitmap: Bitmap) {
val imageView = this.findViewById<ImageView>(R.id.imageView)
imageView.setImageBitmap(bitmap)
}
companion object {
var soundClassifier: SoundClassifier? = null
// fun getSoundClassifier(): SoundClassifier? {
// return soundClassifier
// }
var soundClassifierSpectrogramBased: SoundClassifierSpectrogramBased? = null
var sound_buffer_48000hz = ShortArray(48000 * 10) // Store last 10 seconds in buffer
var sound_buffer_22050hz = ShortArray(22050 * 10) // Store last 10 seconds in buffer
var resampler: Resampler? = null
}
@@ -147,12 +246,12 @@ class MainActivity : AppCompatActivity() {
updateJob = lifecycleScope.launch {
while (isActive) { // isActive is a property of the coroutine scope
updateCounter++
if (last_message_time == 0.toLong())
{
if (last_message_time == 0.toLong()) {
last_message_delay.text = "No messages received"
} else
{
last_message_delay.text = "Last message: ${(Instant.now().toEpochMilli() - last_message_time)/1000F.toInt()} seconds ago"
} else {
last_message_delay.text = "Last message: ${
(Instant.now().toEpochMilli() - last_message_time) / 1000F.toInt()
} seconds ago"
}
// last_message_delay.text = "Update count: $updateCounter"
@@ -171,8 +270,7 @@ class MainActivity : AppCompatActivity() {
val perms = mutableListOf<String>()
if (ContextCompat.checkSelfPermission(
this,
Manifest.permission.ACCESS_COARSE_LOCATION
this, Manifest.permission.ACCESS_COARSE_LOCATION
) != PackageManager.PERMISSION_GRANTED
) {
perms.add(Manifest.permission.ACCESS_COARSE_LOCATION)
@@ -1,16 +1,25 @@
package com.birdsounds.identify
import android.graphics.Bitmap
import android.graphics.Color
import android.util.Log
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import com.theeasiestway.opus.Constants
import com.theeasiestway.opus.Opus
import decodeAACToPCM
import io.github.nailik.androidresampler.Resampler
import io.github.nailik.androidresampler.ResamplerConfiguration
import io.github.nailik.androidresampler.data.ResamplerChannel
import io.github.nailik.androidresampler.data.ResamplerQuality
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.time.Instant
import java.util.LinkedList
import java.util.Queue
var last_message_time = 0L;
@@ -24,6 +33,75 @@ fun ByteArray.toLong(): Long {
return result
}
fun ShortArray.chunked(size: Int): List<ShortArray> {
return this.asList()
.chunked(size)
.map { it.toShortArray() }
}
fun FloatArray.chunked(size: Int): List<FloatArray> {
return this.asList()
.chunked(size)
.map { it.toFloatArray() }
}
fun List<ShortArray>.mergeToShortArray(): ShortArray {
// Calculate total size needed
val totalSize = this.sumOf { it.size }
// Create the output array
val result = ShortArray(totalSize)
// Copy each array into the result
var position = 0
for (array in this) {
array.copyInto(result, position)
position += array.size
}
return result
}
//var sound_buffer_48000hz = ShortArray(48000*10) // Store last 10 seconds in buffer
//var sound_buffer_22050hz = ShortArray(22050*10) // Store last 10 seconds in buffer
var sound_buffer_48000hz: Queue<ShortArray> = LinkedList<ShortArray>();
var sound_buffer_22050hz: Queue<ShortArray> = LinkedList<ShortArray>();
val configuration = ResamplerConfiguration(
quality = ResamplerQuality.BEST,
inputChannel = ResamplerChannel.MONO,
inputSampleRate = 48000,
outputChannel = ResamplerChannel.MONO,
outputSampleRate = 22050
)
val resampler = Resampler(configuration);
fun getPixelValue(i: Int, is_invert: Boolean): Int {
return if (is_invert) 255 - i else i
}
fun getPixelColor(i: Int, is_rgb: Boolean, is_invert: Boolean): Int {
val pixelValue = getPixelValue(i, is_invert);
if (is_rgb) {
return toRgb(pixelValue);
}
return toGrayscale(pixelValue);
}
fun toGrayscale(i: Int): Int {
return Color.rgb(i, i, i);
}
fun toRgb(i: Int): Int {
return Color.argb(255, i, i, i);
}
class MessageListenerService : WearableListenerService() {
@@ -32,9 +110,10 @@ class MessageListenerService : WearableListenerService() {
super.onMessageReceived(p0)
val codec_opus = Opus()
codec_opus.decoderInit(Constants.SampleRate._48000(), Constants.Channels.mono())
// MainActivity
Log.w(TAG, "Data recv: "+p0.data.size.toString() + " bytes")
Log.w(TAG, "Data recv: " + p0.data.size.toString() + " bytes")
val soundclassifier = MainActivity.soundClassifier
val soundClassifierSpectrogramBased = MainActivity.soundClassifierSpectrogramBased;
if (soundclassifier == null) {
Log.w(TAG, "Have invalid sound classifier")
return
@@ -42,6 +121,14 @@ class MessageListenerService : WearableListenerService() {
Log.w(TAG, "Have valid classifier")
}
if (soundClassifierSpectrogramBased == null) {
Log.w(TAG, "Have invalid spectrogram classifier")
return
} else {
Log.w(TAG, "Have valid spectrogram classifier")
}
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
last_message_time = tstamp_bytes.toLong()
@@ -53,8 +140,7 @@ class MessageListenerService : WearableListenerService() {
val buffer = ByteBuffer.wrap(audio_bytes_og)
val sound_a = ByteArrayOutputStream();
val byteArrayList = mutableListOf<ByteArray>()
while (buffer.hasRemaining())
{
while (buffer.hasRemaining()) {
val num_to_read = buffer.get().toInt()
val read_this = ByteArray(num_to_read)
buffer.get(read_this)
@@ -62,40 +148,126 @@ class MessageListenerService : WearableListenerService() {
sound_a.write(decoded);
// Log.e(TAG,"Decompressed ${read_this.size} to ${decoded?.size}")
}
val audio_bytes = sound_a.toByteArray()
codec_opus.decoderRelease();
val short_array = ShortArray(audio_bytes.size/2)
// Log.e(TAG,"Size of short array buffer: "+ decoded?.size.toString());
val audio_bytes = sound_a.toByteArray()
val short_array = ShortArray(audio_bytes.size / 2)
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array)
// Log.e(TAG, pcm_byte_array.sum().toString())
Log.e(TAG, "STARTING SCORING");
var string_send: String = ""
var sorted_list = soundclassifier.executeScoring(short_array)
Log.w(TAG, "FINISHED SCORING");
Log.w(TAG, "")
val threshold = Settings.threshold/100f
for (i in 0 until 10) {
val score = sorted_list[i].value
if (score < threshold) {
continue
val audio_bytes_resampled = resampler.resample(audio_bytes)
if (true) {
val short_array_resampled = ShortArray(audio_bytes_resampled.size / 2)
ByteBuffer.wrap(audio_bytes_resampled).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array_resampled)
val float_array_resampled = short_array_resampled.map { it.toFloat() }.toFloatArray()
val resampled_chunks: List<FloatArray> = float_array_resampled.chunked(512)
val freq_results = mutableListOf<ShortArray>()
for (seg in resampled_chunks) {
val add_this: ShortArray = soundClassifierSpectrogramBased.generateSpectrogram(seg)
freq_results.add(add_this);
// val color_int: List<Int> = add_this.map { Color.rgb(it.toInt(), it.toInt(), it.toInt()) }
// val color_int: List<Int> = add_this.map { Color.argb(125,125, 200, 5) }
// ret_val.add(color_int)
}
val total_points = freq_results.sumOf { it.size }
val freq_array = ShortArray(total_points)
var position = 0;
for (arr in freq_results) {
arr.copyInto(freq_array, position)
position += arr.size
}
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(TAG, species_name + ", " + score.toString())
string_send+= species_name
string_send+=','
string_send+=score.toString()
string_send+=';'
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
var bitmap_pixels = IntArray(freq_array.size);
val createBitmap =
Bitmap.createBitmap(resampled_chunks.size, 128, Bitmap.Config.ARGB_8888);
// Log.d(TAG,"${freq_array.size.toString()} ${createBitmap.width.toString()} ${createBitmap.height.toString()}")
val is_rgb = true;
var is_invert = false;
var height = createBitmap.height
var i = 0
for (i2 in 0 until height) {
val width = createBitmap.getWidth();
var i3 = 0;
while (i3 < width) {
bitmap_pixels[i] =
getPixelColor(
freq_array[(createBitmap.height * i3) + i2].toInt(),
is_rgb,
is_invert
)
i3++;
i++;
}
}
createBitmap.setPixels(
bitmap_pixels,
0,
createBitmap.width,
0,
0,
createBitmap.width,
createBitmap.height
);
soundClassifierSpectrogramBased.run_interpreter(createBitmap)
}
//if (sound_buffer_48000hz.size >= 1) {
var string_send: String = ""
if (true) {
if (true) {
Log.e(TAG, "STARTING SCORING");
val array_score = short_array;
var sorted_list = soundclassifier.executeScoring(array_score)
Log.w(TAG, "FINISHED SCORING");
Log.w(TAG, "")
val threshold = Settings.threshold_waveform / 100f
for (i in 0 until 10) {
val score = sorted_list[i].value
if (score < threshold) {
continue
}
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(TAG, species_name + ", " + score.toString())
string_send += species_name + " ~";
string_send += ','
string_send += score.toString()
string_send += ';'
}
}
if (true) {
string_send+=soundClassifierSpectrogramBased.score_string
}
Log.d(TAG, string_send)
MessageSenderFromPhone.sendMessage(
"/audio",
tstamp_bytes + string_send.toByteArray(), b v
this
)
}
if (sound_buffer_22050hz.size >= 129) {
}
}
}
@@ -1,11 +1,43 @@
package com.birdsounds.identify
object Settings {
var local_model_file: String = "2024_08_16_audio_model.tflite"
var pkg_model_file: String = "2024_08_16/audio-model.tflite"
var local_meta_model_file: String = "2024_08_16_meta_model.tflite"
var pkg_meta_model_file: String = "2024_08_16/meta-model.tflite"
fun create_file_map(pkg_path: String): Map<String, String> {
return mapOf(
"pkg" to pkg_path,
"file" to pkg_path.replace("-", "_").replace("/", "_")
)
}
var threshold: Int = 50;
object Settings {
var threshold_bird_singing: Int = 75
var threshold_spectrogram: Int = 10
var threshold_waveform: Int = 10;
var spectrogram_classify_model_file = create_file_map("from_merlin/sound_id_v40.tflite");
var spectrogram_classify_label_file = create_file_map("from_merlin/sound_id_v40.labels");
var spectrogram_geo_model_file = create_file_map("from_merlin/geo_v40.tflite")
var spectrogram_geo_label_file = create_file_map("from_merlin/geo_v40.labels")
var spectrogram_generate_model_file =
create_file_map("from_merlin/msid685v4_spectrogram.tflite");
var waveform_classify_model_file = create_file_map("2024_08_16/audio-model.tflite");
var waveform_meta_model_file = create_file_map("2024_08_16/meta-model.tflite");
var short_to_long_map = create_file_map("from_merlin/short_to_long.map");
val all_files = listOf(
spectrogram_classify_model_file,
spectrogram_classify_label_file,
spectrogram_geo_model_file,
spectrogram_geo_label_file,
spectrogram_generate_model_file,
waveform_classify_model_file,
waveform_meta_model_file,
short_to_long_map
)
}
@@ -7,6 +7,7 @@ import android.os.SystemClock
import android.preference.PreferenceManager
import android.util.Log
import org.tensorflow.lite.Interpreter
import java.io.BufferedReader
import java.io.File
import java.io.IOException
@@ -99,9 +100,7 @@ class SoundClassifier(
private lateinit var metaInputBuffer: FloatBuffer
private var model_ready = false;
init {;
setupDecoder(context)
loadLabels(context)
loadAssetList(context)
setupInterpreter(context)
setupMetaInterpreter(context)
warmUpModel()
@@ -112,49 +111,12 @@ class SoundClassifier(
{
return this.model_ready;
}
private fun setupDecoder(context: Context) {
}
/** Retrieve asset list from "asset_list" file */
private fun loadAssetList(context: Context) {
try {
val reader =
BufferedReader(InputStreamReader(context.assets.open(options.assetFile)))
val wordList = mutableListOf<String>()
reader.useLines { lines ->
lines.forEach {
wordList.add(it.trim())
}
}
assetList = wordList.map { it }
} catch (e: IOException) {
Log.e(TAG, "Failed to read labels ${options.assetFile}: ${e.message}")
}
}
/** Retrieve labels from "labels.txt" file */
private fun loadLabels(context: Context) {
val localeList = context.resources.configuration.locales
val language = localeList.get(0).language
var filename = options.labelsBase + "_${language}.txt"
var filename = "labels_en.txt"
//Check if file exists
val assetManager = context.assets // Replace 'assets' with actual AssetManager instance
try {
val mapList = assetManager.list("")?.toMutableList()
if (mapList != null) {
if (!mapList.contains(filename)) {
filename = options.labelsBase + "_en.txt"
}
}
} catch (ex: IOException) {
ex.printStackTrace()
filename = options.labelsBase + "_en.txt"
}
Log.i(TAG, filename)
try {
val reader =
BufferedReader(InputStreamReader(context.assets.open(filename)))
@@ -179,7 +141,7 @@ class SoundClassifier(
context.getDir(
"",
Context.MODE_PRIVATE
).absolutePath + "/" + settings.local_model_file;
).absolutePath + "/" + settings.waveform_classify_model_file["file"].toString();
Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
val modelFile = File(modelFilePath)
@@ -190,6 +152,9 @@ class SoundClassifier(
Log.i(TAG, "Done creating TFLite buffer from $modelFilePath")
interpreter = Interpreter(tfliteBuffer, Interpreter.Options())
} catch (e: IOException) {
Log.e(TAG, "Failed to load TFLite model - ${e.message}")
return
@@ -222,7 +187,7 @@ class SoundClassifier(
val metaModelFilePath = context.getDir(
"",
Context.MODE_PRIVATE
).absolutePath + "/" + settings.local_meta_model_file
).absolutePath + "/" + settings.waveform_meta_model_file["file"].toString();
Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath")
val metaModelFile = File(metaModelFilePath)
val tfliteBuffer: ByteBuffer =
@@ -270,6 +235,7 @@ class SoundClassifier(
metaInputBuffer.put(1, lon)
metaInputBuffer.put(2, weekMeta.toFloat())
metaInputBuffer.rewind() // Reset position to beginning of buffer
val metaOutputBuffer = FloatBuffer.allocate(metaModelNumClasses)
metaOutputBuffer.rewind()
meta_interpreter.run(metaInputBuffer, metaOutputBuffer)
@@ -0,0 +1,342 @@
package com.birdsounds.identify
import android.content.Context
import android.graphics.Bitmap
import android.location.Location
import android.media.Image
import android.util.Log
import android.widget.ImageView
import org.apache.commons.math3.stat.descriptive.summary.Sum
import org.tensorflow.lite.Interpreter
import org.tensorflow.lite.support.image.ImageProcessor
import org.tensorflow.lite.support.image.TensorImage
import org.tensorflow.lite.support.image.ops.ResizeOp
import org.tensorflow.lite.support.label.Category
import org.tensorflow.lite.support.tensorbuffer.TensorBuffer
import org.tensorflow.lite.task.core.BaseOptions
import org.tensorflow.lite.task.vision.classifier.Classifications
import org.tensorflow.lite.task.vision.classifier.ImageClassifier
import org.tensorflow.lite.task.vision.classifier.ImageClassifier.ImageClassifierOptions
import java.io.File
import java.nio.ByteBuffer
import java.nio.FloatBuffer
import java.nio.channels.FileChannel
import java.nio.file.StandardOpenOption
import java.time.LocalDate
import java.time.ZoneId
import java.util.Calendar
import java.util.Calendar.WEEK_OF_YEAR
import java.util.Date
var local_species_spectrogram = "";
fun ByteBuffer.toUByteArray(): UByteArray {
// Get remaining bytes in the buffer
val bytes = ByteArray(this.remaining())
// Copy bytes from the buffer
this.get(bytes)
// Convert to UByteArray
return bytes.toUByteArray()
}
class SoundClassifierSpectrogramBased(context_in: MainActivity) {
private var c_threshold: Float = 0.0f;
private var model_ready: Boolean = false;
private var context: MainActivity = context_in;
var norm_factor: Float = (1 shl 15).toFloat();
var score_string = "";
var image_resize_op =
ImageProcessor.Builder().add(ResizeOp(128, 512, ResizeOp.ResizeMethod.BILINEAR)).build();
private lateinit var interpreter_spectrogram: Interpreter;
private var spec_inp_dim: IntArray = intArrayOf(); // [512]
private var spec_out_dim: IntArray = intArrayOf(); // [128,1]
private lateinit var specInputBuffer: TensorBuffer;
private lateinit var specOutputBuffer: TensorBuffer;
private lateinit var interpreter_geo: Interpreter;
private var geo_inp_dim: IntArray = intArrayOf();
private var geo_out_dim: IntArray = intArrayOf();
private lateinit var interpreter_classify: Interpreter;
private lateinit var interpreter_imc: ImageClassifier;
private var classify_inp_dim: IntArray = intArrayOf();
private var classify_out_dim: IntArray = intArrayOf();
private lateinit var classify_input_buffer: FloatBuffer;
private lateinit var classify_input_array: FloatArray;
private lateinit var classify_output_buffer: FloatBuffer;
private lateinit var classify_output_array: FloatArray;
private lateinit var labels_geo: Array<String>
private lateinit var labels_classify: Array<String>
private lateinit var geo_input: FloatBuffer
private lateinit var geo_output: FloatArray
private var geo_prob_map = mutableMapOf<String, Float>();
var short_long_map = mutableMapOf<String, String>()
init {;
loadMetaLabels()
loadLabels()
loadShortLongMap()
setupSpectrogramGenerator()
setupInterpreter()
setupMetaInterpreter()
this.model_ready = true;
}
private fun setupSpectrogramGenerator() {
interpreter_spectrogram =
load_interpreter(Settings.spectrogram_generate_model_file["file"].toString())
var spec_inp = this.interpreter_spectrogram.getInputTensor(/* inputIndex = */ 0)
specInputBuffer = TensorBuffer.createFixedSize(spec_inp.shape(), spec_inp.dataType())
var spec_out = this.interpreter_spectrogram.getOutputTensor(/* inputIndex = */ 0)
specOutputBuffer = TensorBuffer.createFixedSize(spec_out.shape(), spec_out.dataType())
}
fun generateSpectrogram(input_array: FloatArray): ShortArray {
specInputBuffer.buffer.clear()
specOutputBuffer.buffer.clear()
val array_to_load = input_array.copyOf(512).map { it / norm_factor }.toFloatArray()
specInputBuffer.loadArray(array_to_load)
// Log.d(TAG, "Loaded Array: ${array_to_load.min()} ${array_to_load.max()}")
interpreter_spectrogram.run(specInputBuffer.buffer, specOutputBuffer.buffer);
val output = specOutputBuffer.intArray.map { it.toShort() }.toShortArray()
return output
}
private fun setupMetaInterpreter() {
interpreter_geo = load_interpreter(Settings.spectrogram_geo_model_file["file"].toString())
geo_inp_dim = this.interpreter_geo.getInputTensor(0).shape()
Log.i(TAG, "TFLite model input shape: ${geo_inp_dim.contentToString()}")
geo_input = FloatBuffer.allocate(3);
geo_out_dim = this.interpreter_geo.getOutputTensor(0).shape()
Log.i(TAG, "TFLite model input shape: ${geo_out_dim.contentToString()}")
geo_output = FloatArray(geo_out_dim[0]) { 1f };
}
private fun setupInterpreter() {
interpreter_classify =
load_interpreter(Settings.spectrogram_classify_model_file["file"].toString())
interpreter_imc = ImageClassifier.createFromFileAndOptions(
File(get_file_path(Settings.spectrogram_classify_model_file["file"].toString())),
ImageClassifierOptions.builder().setBaseOptions(
BaseOptions.builder().setNumThreads(2).build()
).setMaxResults(10).build()
)
classify_inp_dim = this.interpreter_classify.getInputTensor(0).shape();
Log.i(TAG, "TFLite model input shape: ${classify_inp_dim.contentToString()}")
classify_out_dim = this.interpreter_classify.getOutputTensor(0).shape();
Log.i(TAG, "TFLite model output shape: ${classify_out_dim.contentToString()}")
val n_floats = classify_inp_dim.reduce(Int::times)
classify_input_buffer = FloatBuffer.allocate(n_floats)
classify_input_array = FloatArray(n_floats);
classify_output_buffer = FloatBuffer.allocate(classify_out_dim[1]);
classify_output_array = FloatArray(classify_out_dim[1]) { 0f };
}
private fun loadMetaLabels() {
labels_geo = load_labels(Settings.spectrogram_geo_label_file["file"].toString())
}
private fun loadLabels() {
labels_classify = load_labels(Settings.spectrogram_classify_label_file["file"].toString())
}
private fun loadShortLongMap() {
var labx = load_labels(Settings.short_to_long_map["file"].toString());
for (x in labx) {
var ff = x.split(",")
short_long_map[ff[0]] = ff[1]
}
}
private fun geo_prob_clamp(i: Float): Float {
return when {
i > 0.01 -> 1.0f
i > 0.008 -> 0.8f
i > 0.001 -> 0.5f
else -> 0.0f
}
}
fun runMetaInterpreter(location: Location) {
var localDate: LocalDate = LocalDate.now()
val date: Date = Date.from(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant())
val calendar = Calendar.getInstance()
calendar.time = date
val lon = this.asFloatArray(location.longitude.toFloat());
val lat = this.asFloatArray(location.latitude.toFloat());
val week_of_year = this.asFloatArray(calendar.get(WEEK_OF_YEAR).coerceAtMost(52).toFloat())
val input_array = arrayOf(lon, week_of_year, lat)
var outputTensor = interpreter_geo.getOutputTensor(0);
var fixedTensor =
TensorBuffer.createFixedSize(outputTensor.shape(), outputTensor.dataType())
val outputs: MutableMap<Int, Any> = mutableMapOf()
outputs[0] = fixedTensor.buffer;
interpreter_geo.runForMultipleInputsOutputs(input_array, outputs)
fixedTensor.floatArray.zip(labels_geo).forEach { (score, label) ->
geo_prob_map[label] = geo_prob_clamp(score)
}
val cloned = fixedTensor.floatArray.clone()
val sorted_indices = cloned.withIndex().sortedByDescending { it.value }.map { it.index }
local_species_spectrogram = "Most likely:\n"
for (i in 0..10) {
var c_species = labels_geo[sorted_indices[i]]
local_species_spectrogram += " " + short_long_map.get(c_species).toString();
local_species_spectrogram += "\n";
}
Log.d(TAG, "Likely: " + local_species_spectrogram)
}
fun is_model_ready(): Boolean {
return this.model_ready;
}
private fun get_file_path(file: String): String {
return this.context.getDir(
"", Context.MODE_PRIVATE
).absolutePath + "/" + file.toString();
}
private fun load_labels(file: String): Array<String> {
val label_file_path = this.get_file_path(file);
val labelFile = File(label_file_path)
return labelFile.readLines().toTypedArray()
}
private fun load_interpreter(file: String): Interpreter {
val modelFilePath = get_file_path(file);
val i = Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
val modelFile = File(modelFilePath)
val tfliteBuffer: ByteBuffer =
FileChannel.open(modelFile.toPath(), StandardOpenOption.READ).use { channel ->
channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size())
}
Log.i(TAG, "Done creating TFLite buffer from $modelFilePath")
return Interpreter(tfliteBuffer, Interpreter.Options())
}
private fun asFloatArray(d: Float): FloatArray {
return floatArrayOf(d)
}
fun run_interpreter(createBitmap: Bitmap) {
val fromBitmap = TensorImage.fromBitmap(createBitmap);
val out: TensorImage = image_resize_op.process(fromBitmap);
this.context.runOnUiThread {
this.context.set_image_view(out.bitmap)
}
var classify: MutableList<Classifications> = interpreter_imc.classify(out)
var redone_scores = FloatArray(classify_output_array.size) { 0f };
var cats: MutableList<Category>? = classify.get(0).categories;
score_string = "";
cats?.forEach {
val c_name = it.label
val index = it.index
val score = it.score
var cur_score = 0f;
if (geo_prob_map.containsKey(c_name)) {
cur_score = score * geo_prob_map[c_name]!!
}
if (c_name == "bird1") {
cur_score = score
c_threshold = Settings.threshold_bird_singing / 100f;
}
else
{
c_threshold = Settings.threshold_spectrogram / 100f
}
redone_scores[index] = cur_score
if (cur_score > c_threshold) {
var use_name = "";
if (short_long_map.containsKey(c_name)) {
use_name = short_long_map[c_name].toString()
} else {
use_name = c_name
};
score_string += use_name + "_" + use_name;
score_string += ","
score_string += cur_score.toString();
score_string += ';'
}
}
//
//
// val outList = redone_scores.withIndex().sortedByDescending { it -> it.value };
// for (i in 0 until 6) {
// var cs: IndexedValue<Float> = outList[i]
// Log.d(TAG, i.toString() + " " + labels_classify[cs.index] + " " + cs.value.toString())
// }
//
// Log.d(TAG, classify_output_buffer.toString())
// Log.d(TAG, classify_output_array.sum().toString())
}
}
+143 -12
View File
@@ -12,14 +12,12 @@
android:layout_height="match_parent"
android:orientation="vertical">
<TextView
android:id="@+id/textView"
<ImageView
android:id="@+id/imageView"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:paddingLeft="5dp"
android:paddingTop="15dp"
android:text="Threshold:"
android:textSize="34sp" />
android:layout_height="100dp"
android:background="#ffffff"
android:padding="5dp" />
<LinearLayout
android:layout_width="match_parent"
@@ -27,23 +25,156 @@
android:orientation="horizontal">
<TextView
android:id="@+id/threshold_value_text"
android:layout_width="match_parent"
android:id="@+id/editTextText2"
android:layout_width="35dp"
android:layout_height="match_parent"
android:clipChildren="false"
android:clipToOutline="false"
android:clipToPadding="false"
android:ems="10"
android:gravity="center_horizontal|center_vertical"
android:inputType="text"
android:padding="0dp"
android:paddingHorizontal="0dp"
android:paddingVertical="0dp"
android:paddingStart="0dp"
android:paddingLeft="0dp"
android:paddingTop="0dp"
android:paddingEnd="0dp"
android:paddingRight="0dp"
android:paddingBottom="0dp"
android:rotation="270"
android:singleLine="false"
android:text="Merlin"
android:textAppearance="@style/TextAppearance.AppCompat.Small"
android:textSize="10sp" />
<TextView
android:id="@+id/threshold_value_text_spectrogram"
android:layout_width="76dp"
android:layout_height="match_parent"
android:layout_weight="4"
android:gravity="center"
android:text="Threshold:"
android:textAlignment="center"
android:textSize="28sp" />
<SeekBar
android:id="@+id/threshold_set_scale_bar"
android:id="@+id/threshold_set_scale_bar_spectrogram"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="center|fill_horizontal|fill_vertical"
android:max="100"
android:min="0"
android:progress="20" />
</LinearLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="55dp"
android:orientation="horizontal">
<TextView
android:id="@+id/editTextText5"
android:layout_width="35dp"
android:layout_height="match_parent"
android:clipChildren="false"
android:clipToOutline="false"
android:clipToPadding="true"
android:drawablePadding="0dp"
android:ems="10"
android:gravity="center_horizontal|center_vertical"
android:includeFontPadding="false"
android:inputType="text"
android:padding="0dp"
android:paddingHorizontal="0dp"
android:paddingVertical="0dp"
android:paddingStart="0dp"
android:paddingLeft="0dp"
android:paddingTop="0dp"
android:paddingEnd="0dp"
android:paddingRight="0dp"
android:paddingBottom="0dp"
android:rotation="270"
android:singleLine="false"
android:text="Regular"
android:textAppearance="@style/TextAppearance.AppCompat.Small"
android:textSize="10sp" />
<TextView
android:id="@+id/threshold_value_text_waveform"
android:layout_width="76dp"
android:layout_height="match_parent"
android:gravity="center"
android:text="Threshold:"
android:textAlignment="center"
android:textSize="28sp" />
<SeekBar
android:id="@+id/threshold_set_scale_bar_waveform"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="center|fill_horizontal|fill_vertical"
android:layout_weight="1"
android:max="100"
android:min="0" />
android:min="0"
android:progress="20" />
</LinearLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="55dp"
android:orientation="horizontal">
<TextView
android:id="@+id/editTextText4"
android:layout_width="35dp"
android:layout_height="match_parent"
android:clipChildren="false"
android:clipToOutline="false"
android:clipToPadding="false"
android:ems="10"
android:gravity="center_horizontal|center_vertical"
android:inputType="textMultiLine"
android:lines="2"
android:maxLines="2"
android:minLines="2"
android:padding="0dp"
android:paddingHorizontal="0dp"
android:paddingVertical="0dp"
android:paddingStart="0dp"
android:paddingLeft="0dp"
android:paddingTop="0dp"
android:paddingEnd="0dp"
android:paddingRight="0dp"
android:paddingBottom="0dp"
android:rotation="270"
android:singleLine="false"
android:text="Bird\nHeard"
android:textAppearance="@style/TextAppearance.AppCompat.Small"
android:textSize="10sp" />
<TextView
android:id="@+id/threshold_value_text_birdheard"
android:layout_width="76dp"
android:layout_height="match_parent"
android:gravity="center"
android:text="Threshold:"
android:textAlignment="center"
android:textSize="28sp" />
<SeekBar
android:id="@+id/threshold_set_scale_bar_birdheard"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="center|fill_horizontal|fill_vertical"
android:layout_weight="1"
android:max="100"
android:min="0"
android:progress="80" />
</LinearLayout>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 0 B

-6
View File
@@ -1,6 +0,0 @@
<resources>
<string name="app_name">identify</string>
<string name="title_activity_download">DownloadActivity</string>
<string name="error_download">Download Error</string>
<string name="samples_zero">Samples are all zero</string>
</resources>