thebears/bird_sound_identify_wearos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YACWC

Browse Source
This commit is contained in:
Ishan S. Patel
2025-03-07 21:00:42 -05:00
parent bb17c0651e
commit 65380da39a
32 changed files with 7070 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/deploymentTargetSelector.xml
Generated
+2 -2
View File
@@ -7,10 +7,10 @@
</SelectionState>
<SelectionState runConfigName="wear">
<option name="selectionMode" value="DROPDOWN" />
<DropdownSelection timestamp="2025-03-03T17:08:48.726190400Z">
<DropdownSelection timestamp="2025-03-08T01:58:56.461143900Z">
<Target type="DEFAULT_BOOT">
<handle>
<DeviceId pluginId="PhysicalDevice" identifier="serial=569cc5b0" />
<DeviceId pluginId="Default" identifier="serial=192.168.1.195:44657;connection=82ff69d8" />
</handle>
</Target>
</DropdownSelection>
mobile/build.gradle.kts
+4
View File
@@ -40,6 +40,10 @@ android {
}
dependencies {
api(fileTree("libs") {
include("*.jar")
})
api(files("libs/opus.aar"))
implementation(libs.androidx.core.ktx)
implementation(libs.androidx.appcompat)
implementation(libs.play.services.wearable)
mobile/libs/opus.aar
BIN
View File
Binary file not shown.
mobile/src/main/assets/2024_08_16/audio-model.tflite
BIN
View File
Binary file not shown.
mobile/src/main/assets/2024_08_16/en_us.txt
+6522
View File
File diff suppressed because it is too large Load Diff
mobile/src/main/assets/2024_08_16/meta-model.tflite
BIN
View File
Binary file not shown.
mobile/src/main/assets/metamodel_FP16.tflite
BIN
View File
Binary file not shown.
mobile/src/main/java/com/birdsounds/identify/Downloader.java
-201
View File
@@ -1,201 +0,0 @@
package com.birdsounds.identify;
import android.app.Activity;
import android.content.Context;
import android.util.Log;
import android.view.View;
import android.widget.Toast;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@SuppressWarnings("ResultOfMethodCallIgnored")
public class Downloader {
static final String modelFILE = "modelfx.tflite";
static final String metaModelFILE = "metaModelfx.tflite";
static final String modelURL = "https://raw.githubusercontent.com/woheller69/whoBIRD-TFlite/master/BirdNET_GLOBAL_6K_V2.4_Model_FP16.tflite";
static final String model32URL = "https://raw.githubusercontent.com/woheller69/whoBIRD-TFlite/master/BirdNET_GLOBAL_6K_V2.4_Model_FP32.tflite";
static final String metaModelURL = "https://raw.githubusercontent.com/woheller69/whoBIRD-TFlite/master/BirdNET_GLOBAL_6K_V2.4_MData_Model_FP16.tflite";
static final String modelMD5 = "b1c981fe261910b473b9b7eec9ebcd4e";
static final String model32MD5 = "6c7c42106e56550fc8563adb31bc120e";
static final String metaModelMD5 ="f1a078ae0f244a1ff5a8f1ccb645c805";
public static boolean checkModels(final Activity activity) {
File modelFile = new File(activity.getDir("filesdir", Context.MODE_PRIVATE) + "/" + modelFILE);
File metaModelFile = new File(activity.getDir("filesdir", Context.MODE_PRIVATE) + "/" + metaModelFILE);
String calcModelMD5 = "";
String calcMetaModelMD5 = "";
if (modelFile.exists()) {
try {
byte[] data = Files.readAllBytes(Paths.get(modelFile.getPath()));
byte[] hash = MessageDigest.getInstance("MD5").digest(data);
calcModelMD5 = new BigInteger(1, hash).toString(16);
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
if (metaModelFile.exists()) {
try {
byte[] data = Files.readAllBytes(Paths.get(metaModelFile.getPath()));
byte[] hash = MessageDigest.getInstance("MD5").digest(data);
calcMetaModelMD5 = new BigInteger(1, hash).toString(16);
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
if (modelFile.exists() && !(calcModelMD5.equals(modelMD5) || calcModelMD5.equals(model32MD5))) modelFile.delete();
if (metaModelFile.exists() && !calcMetaModelMD5.equals(metaModelMD5)) metaModelFile.delete();
return (calcModelMD5.equals(modelMD5) || calcModelMD5.equals(model32MD5)) && calcMetaModelMD5.equals(metaModelMD5);
}
public static void downloadModels(final Activity activity) {
File modelFile = new File(activity.getDir("filesdir", Context.MODE_PRIVATE) + "/" + modelFILE);
Log.d("Heyy","Model file checking");
if (!modelFile.exists()) {
Log.d("whoBIRD", "model file does not exist");
Thread thread = new Thread(() -> {
try {
URL url;
if (false) url = new URL(model32URL);
else url = new URL(modelURL);
Log.d("whoBIRD", "Download model");
URLConnection ucon = url.openConnection();
Log.d("whoBIRD", "i am here");
ucon.setReadTimeout(5000);
ucon.setConnectTimeout(10000);
InputStream is = ucon.getInputStream();
BufferedInputStream inStream = new BufferedInputStream(is, 1024 * 5);
modelFile.createNewFile();
FileOutputStream outStream = new FileOutputStream(modelFile);
byte[] buff = new byte[5 * 1024];
int len;
while ((len = inStream.read(buff)) != -1) {
outStream.write(buff, 0, len);
}
outStream.flush();
outStream.close();
inStream.close();
String calcModelMD5="";
if (modelFile.exists()) {
byte[] data = Files.readAllBytes(Paths.get(modelFile.getPath()));
byte[] hash = MessageDigest.getInstance("MD5").digest(data);
calcModelMD5 = new BigInteger(1, hash).toString(16);
} else {
throw new IOException(); //throw exception if there is no modelFile at this point
}
if (!(calcModelMD5.equals(modelMD5) || calcModelMD5.equals(model32MD5) )){
modelFile.delete();
activity.runOnUiThread(() -> {
Toast.makeText(activity, activity.getResources().getString(R.string.error_download), Toast.LENGTH_SHORT).show();
});
} else {
activity.runOnUiThread(() -> {
});
}
} catch (NoSuchAlgorithmException | IOException i) {
activity.runOnUiThread(() -> Toast.makeText(activity, activity.getResources().getString(R.string.error_download), Toast.LENGTH_SHORT).show());
modelFile.delete();
}
});
thread.start();
try {
thread.join();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
Log.d("whoBIRD","model exists");
activity.runOnUiThread(() -> {
});
}
File metaModelFile = new File(activity.getDir("filesdir", Context.MODE_PRIVATE) + "/" + metaModelFILE);
if (!metaModelFile.exists()) {
Log.d("whoBIRD", "meta model file does not exist");
Thread thread = new Thread(() -> {
try {
URL url = new URL(metaModelURL);
Log.d("whoBIRD", "Download meta model");
URLConnection ucon = url.openConnection();
ucon.setReadTimeout(5000);
ucon.setConnectTimeout(10000);
InputStream is = ucon.getInputStream();
BufferedInputStream inStream = new BufferedInputStream(is, 1024 * 5);
metaModelFile.createNewFile();
FileOutputStream outStream = new FileOutputStream(metaModelFile);
byte[] buff = new byte[5 * 1024];
int len;
while ((len = inStream.read(buff)) != -1) {
outStream.write(buff, 0, len);
}
outStream.flush();
outStream.close();
inStream.close();
String calcMetaModelMD5="";
if (metaModelFile.exists()) {
byte[] data = Files.readAllBytes(Paths.get(metaModelFile.getPath()));
byte[] hash = MessageDigest.getInstance("MD5").digest(data);
calcMetaModelMD5 = new BigInteger(1, hash).toString(16);
} else {
throw new IOException(); //throw exception if there is no modelFile at this point
}
if (!calcMetaModelMD5.equals(metaModelMD5)){
metaModelFile.delete();
activity.runOnUiThread(() -> {
Toast.makeText(activity, activity.getResources().getString(R.string.error_download), Toast.LENGTH_SHORT).show();
});
} else {
activity.runOnUiThread(() -> {
});
}
} catch (NoSuchAlgorithmException | IOException i) {
activity.runOnUiThread(() -> Toast.makeText(activity, activity.getResources().getString(R.string.error_download), Toast.LENGTH_SHORT).show());
metaModelFile.delete();
Log.w("whoBIRD", activity.getResources().getString(R.string.error_download), i);
}
});
thread.start();
try {
thread.join();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
Log.d("whoBIRD", "meta file exists");
activity.runOnUiThread(() -> {
});
}
}
}
mobile/src/main/java/com/birdsounds/identify/Location.java
-59
View File
@@ -1,59 +0,0 @@
package com.birdsounds.identify;
import android.Manifest;
import android.content.Context;
import android.content.pm.PackageManager;
import android.location.LocationListener;
import android.location.LocationManager;
import android.os.Bundle;
import android.widget.Toast;
import androidx.core.app.ActivityCompat;
public class Location {
private static LocationListener locationListenerGPS;
static void stopLocation(Context context){
LocationManager locationManager = (LocationManager) context.getSystemService(Context.LOCATION_SERVICE);
if (locationListenerGPS!=null) locationManager.removeUpdates(locationListenerGPS);
locationListenerGPS=null;
}
static void requestLocation(Context context, SoundClassifier soundClassifier) {
if (ActivityCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == PackageManager.PERMISSION_GRANTED && checkLocationProvider(context)) {
LocationManager locationManager = (LocationManager) context.getSystemService(Context.LOCATION_SERVICE);
if (locationListenerGPS==null) locationListenerGPS = new LocationListener() {
@Override
public void onLocationChanged(android.location.Location location) {
soundClassifier.runMetaInterpreter(location);
}
@Deprecated
@Override
public void onStatusChanged(String provider, int status, Bundle extras) {
}
@Override
public void onProviderEnabled(String provider) {
}
@Override
public void onProviderDisabled(String provider) {
}
};
locationManager.requestLocationUpdates(LocationManager.GPS_PROVIDER, 60000, 0, locationListenerGPS);
}
}
public static boolean checkLocationProvider(Context context) {
LocationManager locationManager = (LocationManager) context.getSystemService(Context.LOCATION_SERVICE);
if (!locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)){
Toast.makeText(context, "Error no GPS", Toast.LENGTH_SHORT).show();
return false;
} else {
return true;
}
}
}
mobile/src/main/java/com/birdsounds/identify/MessageListenerService.kt
-57
View File
@@ -1,57 +0,0 @@
package com.birdsounds.identify
import android.content.Intent
import android.util.Half.abs
import android.util.Log
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.ShortBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// MainActivity
val soundclassifier = MainActivity.soundClassifier
if (soundclassifier == null) {
Log.w(tag, "Have invalid sound classifier")
return
} else {
Log.w(tag, "Have valid classifier")
}
val short_array = ShortArray(48000 * 3)
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
var string_send: String = ""
ByteBuffer.wrap(audio_bytes).order(
ByteOrder.LITTLE_ENDIAN
).asShortBuffer().get(short_array)
Log.w(tag, short_array.sum().toString())
var sorted_list = soundclassifier.executeScoring(short_array)
Log.w(tag, "")
for (i in 0 until 5) {
val score = sorted_list[i].value
val index = sorted_list[i].index
val species_name = soundclassifier.labelList[index]
Log.w(tag, species_name + ", " + score.toString())
string_send+= species_name
string_send+=','
string_send+=score.toString()
string_send+=';'
}
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
// Log.i(tag , short_array.map( { abs(it)}).sum().toString())
// Log.i(tag, short_array[0].toString())
// Log.i(tag, p0.data.toString(Charsets.US_ASCII))
// broadcastMessage(p0)
}
}
mobile/src/main/java/com/birdsounds/identify/phone_AudioDecoder.kt
+101
View File
@@ -0,0 +1,101 @@
import android.content.ContentValues.TAG
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaFormat
import android.util.Log
import java.nio.ByteBuffer
fun listMediaCodecDecoders() {
val codecList = MediaCodecList(MediaCodecList.ALL_CODECS) // Get all codecs
val codecs = codecList.codecInfos
Log.e(TAG, "Available MediaCodec Decoders:")
for (codec in codecs) {
if (!codec.isEncoder) { // Check if the codec is a decoder
Log.e(TAG, "Decoder: ${codec.name}")
// List the MIME types supported by the decoder
val supportedTypes = codec.supportedTypes
Log.e(TAG, " Supported Types:")
for (type in supportedTypes) {
Log.e(TAG, " $type")
}
}
}
}
fun decodeAACToPCM(inputData: ByteArray): ShortArray {
// listMediaCodecDecoders();
// Media format configuration for AAC
val mediaFormat = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_OPUS, // MIME type for AAC
48000, // Sample rate, change this based on your input data
1 // Channel count, change this based on your input data
)
// mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 64000) // 128kbps
// mediaFormat.setInteger(MediaFormat.KEY_IS_ADTS, 0) // AAC should use ADTS header
// mediaFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
// Create a decoder for AAC
val mediaCodec = MediaCodec.createDecoderByType( MediaFormat.MIMETYPE_AUDIO_OPUS);
mediaCodec.configure(mediaFormat, null, null, 0)
mediaCodec.start()
val decodedSamples = mutableListOf<Short>()
// Variables for handling input and output buffers
val bufferInfo = MediaCodec.BufferInfo()
var inputOffset = 0;
while (inputOffset < inputData.size || true) {
// Feed input data to the codec
val inputBufferIndex = mediaCodec.dequeueInputBuffer(100000) // Timeout in microseconds
if (inputBufferIndex >= 0 && inputOffset < inputData.size) {
val inputBuffer: ByteBuffer? = mediaCodec.getInputBuffer(inputBufferIndex)
inputBuffer?.clear()
// Calculate the number of bytes to write to the buffer
val chunkSize = kotlin.math.min(inputBuffer?.capacity() ?: 0, inputData.size - inputOffset)
Log.e(TAG, "Chunk size: " + chunkSize.toString())
inputBuffer?.put(inputData, inputOffset, chunkSize)
inputOffset += chunkSize
// Pass the data to the codec
mediaCodec.queueInputBuffer(inputBufferIndex, 0, chunkSize, 0, 0)
}
// Process output data
val outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 10000)
Log.e(TAG, "Output buffer index: " + outputBufferIndex.toString())
if (outputBufferIndex >= 0) {
val outputBuffer: ByteBuffer? = mediaCodec.getOutputBuffer(outputBufferIndex)
// Convert byte buffer to PCM data (16-bit integers)
val pcmData = ShortArray(bufferInfo.size / 2)
outputBuffer?.asShortBuffer()?.get(pcmData)
// Add PCM data to the final output array
decodedSamples.addAll(pcmData.toList())
// Release the output buffer
mediaCodec.releaseOutputBuffer(outputBufferIndex, false)
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// Handle format changes, if needed
mediaCodec.outputFormat
} else if (outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// Output buffer not available, retry later
if (inputOffset >= inputData.size) break
}
}
// Release the codec when done
mediaCodec.stop()
mediaCodec.release()
return decodedSamples.toShortArray()
}
mobile/src/main/java/com/birdsounds/identify/phone_Downloader.kt
+52
View File
@@ -0,0 +1,52 @@
package com.birdsounds.identify
import android.content.Context
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
class Downloader(mainActivity: MainActivity) {
private val settings = Settings();
private var activity: MainActivity = mainActivity;
private var context: Context = activity.applicationContext;
fun copyAssetToFolder(assetName: String, destinationPath: String): Boolean {
try {
// Get the input stream from the asset
val assetInputStream = context.assets.open(assetName)
// Create the destination directory if it doesn't exist
val destinationFile = File(activity.getDir("",Context.MODE_PRIVATE).absolutePath + "/" + destinationPath)
destinationFile.parentFile?.mkdirs()
// Copy the file
val buffer = ByteArray(1024)
val outputStream = FileOutputStream(destinationFile)
var read: Int
while (assetInputStream.read(buffer).also { read = it } != -1) {
outputStream.write(buffer, 0, read)
}
assetInputStream.close()
outputStream.flush()
outputStream.close()
return true
} catch (e: IOException) {
e.printStackTrace()
return false
}
}
fun prepareModelFiles()
{
copyAssetToFolder(settings.pkg_model_file, settings.local_model_file);
copyAssetToFolder(settings.pkg_meta_model_file, settings.local_meta_model_file);
}
}
mobile/src/main/java/com/birdsounds/identify/phone_Location.kt
+71
View File
@@ -0,0 +1,71 @@
package com.birdsounds.identify
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.location.Location
import android.location.LocationListener
import android.location.LocationManager
import android.os.Bundle
import android.util.Log
import android.widget.Toast
import androidx.core.app.ActivityCompat
object Location {
private var locationListenerGPS: LocationListener? = null
fun stopLocation(context: Context) {
val locationManager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
if (locationListenerGPS != null) locationManager.removeUpdates(
locationListenerGPS!!
)
locationListenerGPS = null
}
fun requestLocation(context: Context, soundClassifier: SoundClassifier) {
if (ActivityCompat.checkSelfPermission(
context,
Manifest.permission.ACCESS_COARSE_LOCATION
) == PackageManager.PERMISSION_GRANTED && checkLocationProvider(context)
) {
val locationManager =
context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
if (locationListenerGPS == null) locationListenerGPS = object : LocationListener {
override fun onLocationChanged(location: Location) {
Log.w(TAG, "Got location changed");
while (!soundClassifier.is_model_ready()) {
Thread.sleep(50);
}
Log.w(TAG, "Sound classifier is ready");
soundClassifier.runMetaInterpreter(location)
soundClassifier.runMetaInterpreter(location)
}
@Deprecated("")
override fun onStatusChanged(provider: String, status: Int, extras: Bundle) {
}
override fun onProviderEnabled(provider: String) {
}
override fun onProviderDisabled(provider: String) {
}
}
locationManager.requestLocationUpdates(
LocationManager.GPS_PROVIDER, 60000, 0f,
locationListenerGPS!!
)
}
}
fun checkLocationProvider(context: Context): Boolean {
val locationManager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
if (!locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) {
Toast.makeText(context, "Error no GPS", Toast.LENGTH_SHORT).show()
return false
} else {
return true
}
}
}
mobile/src/main/java/com/birdsounds/identify/MainActivity.kt → mobile/src/main/java/com/birdsounds/identify/phone_MainActivity.kt
+14 -6
View File
@@ -3,7 +3,6 @@ package com.birdsounds.identify
import android.content.pm.PackageManager
import android.os.Bundle
import android.Manifest
import android.annotation.SuppressLint
import android.util.Log
import androidx.activity.enableEdgeToEdge
import androidx.appcompat.app.AppCompatActivity
@@ -13,6 +12,13 @@ import androidx.core.view.WindowInsetsCompat
import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
val Any.TAG: String
get() {
val tag = javaClass.simpleName
return if (tag.length <= 23) tag else tag.substring(0, 23)
}
class MainActivity : AppCompatActivity() {
// private lateinit var soundClassifier: SoundClassifier
val REQUEST_PERMISSIONS = 1337
@@ -26,15 +32,17 @@ class MainActivity : AppCompatActivity() {
.registerChannelCallback(object : ChannelClient.ChannelCallback() {
override fun onChannelOpened(channel: ChannelClient.Channel) {
super.onChannelOpened(channel)
Log.d("HEY", "onChannelOpened")
Log.d(TAG, "onChannelOpened")
}
}
)
Downloader.downloadModels(this)
Downloader(this).prepareModelFiles();
Log.w(TAG, "Finished setting up downloader")
requestPermissions()
soundClassifier = SoundClassifier(this, SoundClassifier.Options())
Location.requestLocation(this, soundClassifier)
Log.w(TAG, "Starting sound classifier")
Location.requestLocation(this, soundClassifier!!)
Log.w(TAG, "Starting location requester")
ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets ->
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom)
@@ -60,6 +68,6 @@ class MainActivity : AppCompatActivity() {
perms.add(Manifest.permission.ACCESS_COARSE_LOCATION)
perms.add(Manifest.permission.ACCESS_FINE_LOCATION)
}
if (!perms.isEmpty()) requestPermissions(perms.toTypedArray(), REQUEST_PERMISSIONS)
if (perms.isNotEmpty()) requestPermissions(perms.toTypedArray(), REQUEST_PERMISSIONS)
}
}
mobile/src/main/java/com/birdsounds/identify/MessageConstants.kt → mobile/src/main/java/com/birdsounds/identify/phone_MessageConstants.kt
-1
View File
@@ -1,5 +1,4 @@
package com.birdsounds.identify
import android.content.Intent
object MessageConstants {
const val intentName = "WearableMessageDisplay"
mobile/src/main/java/com/birdsounds/identify/phone_MessageListenerService.kt
+56
View File
@@ -0,0 +1,56 @@
package com.birdsounds.identify
import android.util.Log
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import decodeAACToPCM
class MessageListenerService : WearableListenerService() {
// fun placeSoundClassifier(soundClassifier: SoundClassifier)
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// MainActivity
Log.w(TAG, "Data recv: "+p0.data.size.toString() + " bytes")
val soundclassifier = MainActivity.soundClassifier
if (soundclassifier == null) {
Log.w(TAG, "Have invalid sound classifier")
return
} else {
Log.w(TAG, "Have valid classifier")
}
var tstamp_bytes = p0.data.copyOfRange(0, Long.SIZE_BYTES)
var audio_bytes = p0.data.copyOfRange(Long.SIZE_BYTES, p0.data.size)
var string_send: String = ""
val pcm_byte_array = decodeAACToPCM(audio_bytes)
Log.e(TAG,"Size of short array buffer: "+ pcm_byte_array.size.toString());
// ByteBuffer.wrap(audio_bytes).order(
// ByteOrder.LITTLE_ENDIAN
// ).asShortBuffer().get(short_array)
Log.e(TAG, pcm_byte_array.sum().toString())
Log.e(TAG, "STARTING SCORING");
// var sorted_list = soundclassifier.executeScoring(short_array)
// Log.w(TAG, "FINISHED SCORING");
// Log.w(TAG, "")
// for (i in 0 until 5) {
// val score = sorted_list[i].value
// val index = sorted_list[i].index
// val species_name = soundclassifier.labelList[index]
// Log.w(TAG, species_name + ", " + score.toString())
// string_send+= species_name
// string_send+=','
// string_send+=score.toString()
// string_send+=';'
// }
MessageSenderFromPhone.sendMessage("/audio", tstamp_bytes + string_send.toByteArray(), this)
}
}
mobile/src/main/java/com/birdsounds/identify/MessageSenderFromPhone.kt → mobile/src/main/java/com/birdsounds/identify/phone_MessageSenderFromPhone.kt
View File
mobile/src/main/java/com/birdsounds/identify/phone_Settings.kt
+9
View File
@@ -0,0 +1,9 @@
package com.birdsounds.identify
class Settings {
var local_model_file: String = "2024_08_16_audio_model.tflite"
var pkg_model_file: String = "2024_08_16/audio-model.tflite"
var local_meta_model_file: String = "2024_08_16_meta_model.tflite"
var pkg_meta_model_file: String = "2024_08_16/meta-model.tflite"
}
mobile/src/main/java/com/birdsounds/identify/SoundClassifier.kt → mobile/src/main/java/com/birdsounds/identify/phone_SoundClassifier.kt
+24 -17
View File
@@ -1,11 +1,11 @@
package com.birdsounds.identify
import android.content.Context
import android.location.Location
import android.os.SystemClock
import android.preference.PreferenceManager
import android.util.Log
import androidx.annotation.Nullable
import org.tensorflow.lite.Interpreter
import java.io.BufferedReader
import java.io.File
@@ -23,8 +23,6 @@ import kotlin.concurrent.scheduleAtFixedRate
import kotlin.math.ceil
import kotlin.math.cos
import uk.me.berndporr.iirj.Butterworth
import java.nio.ShortBuffer
import kotlin.math.round
import kotlin.math.sin
@@ -32,7 +30,11 @@ class SoundClassifier(
context: Context,
private val options: Options = Options()
) {
internal var mContext: Context
val TAG = "Sound Classifier"
init {
@@ -40,14 +42,10 @@ class SoundClassifier(
}
class Options(
/** Path of the converted model label file, relative to the assets/ directory. */
val labelsBase: String = "labels",
/** Path of the converted .tflite file, relative to the assets/ directory. */
val assetFile: String = "assets.txt",
/** Path of the converted .tflite file, relative to the assets/ directory. */
val modelPath: String = "modelfx.tflite",
/** Path of the meta model .tflite file, relative to the assets/ directory. */
val metaModelPath: String = "metaModelfx.tflite",
/** The required audio sample rate in Hz. */
val sampleRate: Int = 48000,
/** Multiplier for audio samples */
@@ -83,7 +81,7 @@ class SoundClassifier(
/** Number of output classes of the TFLite model. */
private var modelNumClasses = 0
private var metaModelNumClasses = 0
private var settings: Settings = Settings();
/** Used to hold the real-time probabilities predicted by the model for the output classes. */
private lateinit var predictionProbs: FloatArray
@@ -94,19 +92,26 @@ class SoundClassifier(
private var recognitionTask: TimerTask? = null
/** Buffer that holds audio PCM sample that are fed to the TFLite model for inference. */
private lateinit var inputBuffer: FloatBuffer
private lateinit var metaInputBuffer: FloatBuffer
init {
private var model_ready = false;
init {;
setupDecoder(context)
loadLabels(context)
loadAssetList(context)
setupInterpreter(context)
setupMetaInterpreter(context)
warmUpModel()
this.model_ready = true;
}
fun is_model_ready(): Boolean
{
return this.model_ready;
}
private fun setupDecoder(context: Context) {
}
/** Retrieve asset list from "asset_list" file */
private fun loadAssetList(context: Context) {
@@ -168,10 +173,12 @@ class SoundClassifier(
private fun setupInterpreter(context: Context) {
try {
val modelFilePath = context.getDir(
"filesdir",
val modelFilePath =
context.getDir(
"",
Context.MODE_PRIVATE
).absolutePath + "/" + options.modelPath
).absolutePath + "/" + settings.local_model_file;
Log.i(TAG, "Trying to create TFLite buffer from $modelFilePath")
val modelFile = File(modelFilePath)
val tfliteBuffer: ByteBuffer =
@@ -211,9 +218,9 @@ class SoundClassifier(
try {
val metaModelFilePath = context.getDir(
"filesdir",
"",
Context.MODE_PRIVATE
).absolutePath + "/" + options.metaModelPath
).absolutePath + "/" + settings.local_meta_model_file
Log.i(TAG, "Trying to create TFLite buffer from $metaModelFilePath")
val metaModelFile = File(metaModelFilePath)
val tfliteBuffer: ByteBuffer =
@@ -244,6 +251,7 @@ class SoundClassifier(
}
// Fill the array with 1 initially.
metaPredictionProbs = FloatArray(metaModelNumClasses) { 1f }
metaInputBuffer = FloatBuffer.allocate(metaModelInputLength)
}
@@ -333,7 +341,6 @@ class SoundClassifier(
else inputBuffer.put(i, butterworth.filter(s.toDouble()).toFloat())
}
inputBuffer.rewind()
outputBuffer.rewind()
interpreter.run(inputBuffer, outputBuffer)
wear/build.gradle.kts
+6 -1
View File
@@ -39,6 +39,11 @@ android {
}
dependencies {
api(fileTree("libs") {
include("*.jar")
})
api(files("libs/opus.aar"))
implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.4")
implementation("androidx.compose.ui:ui-tooling:1.3.1")
implementation("androidx.navigation:navigation-compose:2.8.0-rc01")
@@ -47,7 +52,7 @@ dependencies {
implementation("com.google.android.horologist:horologist-audio:0.6.18")
implementation("com.google.android.horologist:horologist-compose-tools:0.6.18")
implementation("com.google.android.horologist:horologist-compose-tools:0.6.18")
implementation("com.google.android.horologist:horologist-compose-layout:0.6.18")
implementation("com.google.android.horologist:horolo+++_gist-compose-layout:0.6.18")
implementation("androidx.compose.material:material-icons-core:1.6.8")
implementation("androidx.compose.material:material-icons-extended:1.6.8")
implementation("com.google.android.horologist:horologist-compose-material:0.6.8")
wear/libs/opus.aar
BIN
View File
Binary file not shown.
wear/src/main/java/com/birdsounds/identify/presentation/MessageListenerService.kt
-28
View File
@@ -1,28 +0,0 @@
package com.birdsounds.identify.presentation
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
import java.nio.ByteBuffer
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
val t_scored = ByteBuffer.wrap(p0.data).getLong()
var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
var score_species_string = byte_strings.decodeToString()
var list_strings: List<String> = score_species_string.split(';')
list_strings.map({
var split_str = it.split(',')
if (split_str.size == 2) {
var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
if (out.score > 0.05) {
SpeciesList.add_observation(out)
}
}
})
MessageSender.messageLog.add(t_scored)
}
}
wear/src/main/java/com/birdsounds/identify/presentation/watch_AudioEncoder.kt
+140
View File
@@ -0,0 +1,140 @@
import android.content.ContentValues.TAG
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaFormat
import android.util.Log
/**
* Encodes PCM audio data to AAC format
* @param pcmData The raw PCM audio data to encode
* @param sampleRate Sample rate of the audio (e.g., 44100)
* @param channelCount Number of audio channels (1 for mono, 2 for stereo)
* @return ByteArray containing the encoded AAC audio data
*/
fun listMediaCodecEncoders() {
val codecList = MediaCodecList(MediaCodecList.ALL_CODECS) // List all codecs
val codecs = codecList.codecInfos
println("Available MediaCodec Encoders:")
for (codec in codecs) {
if (codec.isEncoder) {
Log.e(TAG, "Encoder: ${codec.name}")
// List supported types for this encoder
val supportedTypes = codec.supportedTypes
Log.w(TAG, " Supported Types:")
for (type in supportedTypes) {
Log.w(TAG, " $type")
}
}
}
}
fun encodePcmToAac(pcmData: ByteArray): ByteArray {
// Create a format for the encoder
var sampleRate = 48000
var channelCount = 1
// listMediaCodecEncoders();
val format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_OPUS, sampleRate, channelCount)
format.setInteger(MediaFormat.KEY_BIT_RATE, 64000) // 128kbps
// format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
// format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, pcmData.size)
// Create and configure the encoder
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS)
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
codec.start()
// Use ByteArrayOutputStream to collect encoded data
val encodedBytes = mutableListOf<Byte>()
val bufferInfo = MediaCodec.BufferInfo()
var allInputSubmitted = false
var inputOffset = 0
var presentationTimeUs = 0L
val frameSize = 1024 * channelCount * 2 // Typical frame size for AAC encoding (1024 samples, 16-bit PCM)
try {
while (!allInputSubmitted || bufferInfo.flags != MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
// Submit input data to encoder
if (!allInputSubmitted) {
val inputBufferId = codec.dequeueInputBuffer(10000)
if (inputBufferId >= 0) {
val inputBuffer = codec.getInputBuffer(inputBufferId)
inputBuffer?.clear()
// Calculate how many bytes to read
val bytesToRead = if (inputOffset < pcmData.size) {
minOf(inputBuffer!!.capacity(), pcmData.size - inputOffset)
} else {
0
}
if (bytesToRead > 0) {
// Copy data from byte array to input buffer
inputBuffer!!.put(pcmData, inputOffset, bytesToRead)
inputOffset += bytesToRead
// Calculate presentation time in microseconds
// (samples / sample rate) * 1_000_000
val samples = bytesToRead / (2 * channelCount) // 16-bit samples
presentationTimeUs += samples * 1_000_000L / sampleRate
codec.queueInputBuffer(
inputBufferId,
0,
bytesToRead,
presentationTimeUs,
0
)
} else {
// End of input data
codec.queueInputBuffer(
inputBufferId,
0,
0,
presentationTimeUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
allInputSubmitted = true
}
}
}
// Get encoded data from encoder
val outputBufferId = codec.dequeueOutputBuffer(bufferInfo, 10000)
when {
outputBufferId >= 0 -> {
val outputBuffer = codec.getOutputBuffer(outputBufferId)
if (outputBuffer != null && bufferInfo.size > 0) {
// Copy encoded data to our result buffer
val encodedChunk = ByteArray(bufferInfo.size)
outputBuffer.position(bufferInfo.offset)
outputBuffer.limit(bufferInfo.offset + bufferInfo.size)
outputBuffer.get(encodedChunk)
encodedBytes.addAll(encodedChunk.toList())
}
codec.releaseOutputBuffer(outputBufferId, false)
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
break
}
}
outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
// You might want to store the format for muxing
val newFormat = codec.outputFormat
// Log.d("MediaCodec", "Output format changed: $newFormat")
}
}
}
} finally {
codec.stop()
codec.release()
}
return encodedBytes.toByteArray()
}
wear/src/main/java/com/birdsounds/identify/presentation/ControlDashboard.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_ControlDashboard.kt
View File
wear/src/main/java/com/birdsounds/identify/presentation/MainActivity.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_MainActivity.kt
View File
wear/src/main/java/com/birdsounds/identify/presentation/MainState.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_MainState.kt
View File
wear/src/main/java/com/birdsounds/identify/presentation/watch_MessageListenerService.kt
+27
View File
@@ -0,0 +1,27 @@
package com.birdsounds.identify.presentation
import com.google.android.gms.wearable.MessageEvent
import com.google.android.gms.wearable.WearableListenerService
class MessageListenerService : WearableListenerService() {
private val tag = "MessageListenerService"
override fun onMessageReceived(p0: MessageEvent) {
super.onMessageReceived(p0)
// val t_scored = ByteBuffer.wrap(p0.data).getLong()
// var byte_strings: ByteArray = p0.data.copyOfRange(8, p0.data.size)
// var score_species_string = byte_strings.decodeToString()
// var list_strings: List<String> = score_species_string.split(';')
// list_strings.map({
// var split_str = it.split(',')
// if (split_str.size == 2) {
// var out = AScore(split_str[0], split_str[1].toFloat(), t_scored)
// if (out.score > 0.05) {
// SpeciesList.add_observation(out)
// }
// }
// })
// MessageSender.messageLog.add(t_scored)
}
}
wear/src/main/java/com/birdsounds/identify/presentation/MessageSender.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_MessageSender.kt
+23 -2
View File
@@ -10,24 +10,32 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import java.io.ByteArrayOutputStream
import java.util.concurrent.ExecutionException
import java.util.zip.GZIPOutputStream
object MessageSender {
const val tag = "MessageSender"
private val job = Job()
private val coroutineScope = CoroutineScope(Dispatchers.IO + job)
var messageLog = ConcurrentMutableSet<Long>()
fun sendMessage(path: String, message: ByteArray, context: Context) {
coroutineScope.launch {
sendMessageInBackground(path, message, context)
}
}
fun compressByteArray(input: ByteArray): ByteArray {
val outputStream = ByteArrayOutputStream()
GZIPOutputStream(outputStream).use { gzip ->
gzip.write(input) // Compress the byte array
}
return outputStream.toByteArray() // Return the compressed data
}
private fun sendMessageInBackground(path: String, message: ByteArray, context: Context) {
//first get all the nodes, ie connected wearable devices.
val nodeListTask = Wearable.getNodeClient(context).connectedNodes
@@ -38,6 +46,19 @@ object MessageSender {
if(nodes.isEmpty()) {
Log.i(tag,"No Node found to send message")
}
// var compressed_message = audio_encoder.encodePCMToAAC(message)
// Log.w(tag, "Uncompressed message size "+message.size.toString())
// Log.w(tag, "Compressed message size "+compressed_message.size.toString())
//Now send the message to each device.
for (node in nodes) {
val sendMessageTask = Wearable.getMessageClient(context)
wear/src/main/java/com/birdsounds/identify/presentation/SoundRecorder.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_SoundRecorder.kt
+19 -21
View File
@@ -1,5 +1,6 @@
package com.birdsounds.identify.presentation
import com.theeasiestway.opus.Constants
import com.theeasiestway.opus.Opus
import android.Manifest
import android.content.Context
@@ -8,6 +9,7 @@ import android.media.AudioRecord
import android.media.MediaRecorder
import android.util.Log
import androidx.annotation.RequiresPermission
import encodePcmToAac
import kotlinx.coroutines.suspendCancellableCoroutine
import java.nio.ByteBuffer
import java.time.Instant
@@ -21,6 +23,7 @@ class SoundRecorder(
outputFileName: String
) {
private val codec = Opus();
private var state = State.IDLE
private var context = context_in
@@ -63,41 +66,32 @@ class SoundRecorder(
val thread = Thread {
// var sent_first: Boolean = false
var ignore_warmup: Boolean = true
var num_chunked_since_last_send = 0
var last_tstamp: Long = Instant.now().toEpochMilli()
var do_send_message: Boolean = false
while (true) {
var last_tstamp: Long = Instant.now().toEpochMilli();
while (true) /**/{
if (Thread.interrupted()) {
// check for the interrupted flag, reset it, and throw exception
Log.w(TAG, "Finished thread")
break
}
chunk_index = chunk_index.mod(num_chunks)
if (chunk_index == 0) {
codec.encoderInit(48000, 1, Constants.Application.audio());
}
val out = audioRecord.read(
/* audioData = */ chunked_audio_bytes[chunk_index],
/* offsetInBytes = */ 0,
/* sizeInBytes = */ chunk_size,
/* readMode = */ AudioRecord.READ_BLOCKING
)
num_chunked_since_last_send += 1
do_send_message = false
if (num_chunked_since_last_send >= num_chunks) {
do_send_message = true
Log.w("MSG","sending message because full 3s have passed")
} else if ((last_tstamp in MessageSender.messageLog) && (num_chunked_since_last_send>4)) {
do_send_message = true
Log.w("MSG","Send message because the phone has finished")
} else if ((ignore_warmup) && (num_chunked_since_last_send > 2)) {
do_send_message = true
Log.w("MSG","Sent message because ignoring warmup")
}
chunk_index += 1
if ((do_send_message)) {
var tstamp: Long = Instant.now().toEpochMilli()
val tstamp_buffer = ByteBuffer.allocate(Long.SIZE_BYTES)
val tstamp_bytes = tstamp_buffer.putLong(tstamp).array()
@@ -111,9 +105,13 @@ class SoundRecorder(
byte_send += chunked_audio_bytes[c_index]
}
// do_send_message = false;
num_chunked_since_last_send = 0
// num_chunked_since_last_send = 0
// ignore_warmup = false;
MessageSender.messageLog.clear()
MessageSender.sendMessage("/audio", tstamp_bytes + byte_send, context)
val compressed = encodePcmToAac(byte_send)
Log.i(TAG,"Size pre-compression "+byte_send.size.toString())
Log.i(TAG,"Size post-compression "+compressed.size.toString())
MessageSender.sendMessage("/audio", compressed, context)
last_tstamp = tstamp
}
wear/src/main/java/com/birdsounds/identify/presentation/SpeciesList.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_SpeciesList.kt
-2
View File
@@ -61,8 +61,6 @@ object SpeciesList {
} else {
internal_list.add(species_in)
}
internal_list = internal_list.sortedBy({ (it.age()) }).toMutableList()
wear/src/main/java/com/birdsounds/identify/presentation/SpeciesListView.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_SpeciesListView.kt
View File
wear/src/main/java/com/birdsounds/identify/presentation/StartRecordingScreen.kt → wear/src/main/java/com/birdsounds/identify/presentation/watch_StartRecordingScreen.kt
View File