Runtime Voice Intents Crash Unity Application on Command Recognition

audeo · May 30, 2023, 9:55pm

I am building a simple application based on this Voice Intents example. I've configured Unity as the setup guide directs, and have built a script attached to an empty game object that reads a list of commands from a file and sets up the voice commands. Here is the script that does this:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;
using System.IO;
using UnityEngine.XR.MagicLeap;
using UnityEngine.Android;

public class VoiceCommandConfigReader : MonoBehaviour
{
    // Useful Structures
    public enum ConfigType
    {
        JSON,
        XML,
        YAML,
        CSV
    };
    public struct VoiceCommandConfig
    {
        public string command;
    };

    // Delegates
    public delegate void PrintStringEvent(in string message);
    static public event PrintStringEvent printStringEvent;

    // Public Members
    public ConfigType configFileType;
    public string configFileName;

    // Private Members
    private string configFilePath;
    private MLVoiceIntentsConfiguration _voiceConfiguration;

    // Permissions Callbacks
    private MLPermissions.Callbacks callbacks;

    // Start is called before the first frame update
    void Start()
    {
        // Link Permissions Callbacks
        //callbacks.OnPermissionGranted += onPermissionGranted;
        //callbacks.OnPermissionDenied += onPermissionDenied;
        //callbacks.OnPermissionDeniedAndDontAskAgain += onPermissionDenied;


    #if UNITY_EDITOR
        configFilePath = Application.dataPath + "/" + configFileName;

#elif UNITY_ANDROID
        configFilePath = Application.persistentDataPath + "/" + configFileName; //TODO: is this really the best way to do this?
#endif


        MLPermissions.RequestPermission(MLPermission.VoiceInput, callbacks);
        MLPermissions.RequestPermission(MLPermission.ReadExternalStorage, callbacks);
        MLPermissions.RequestPermission(MLPermission.WriteExternalStorage, callbacks);

        
        if (MLPermissions.CheckPermission(MLPermission.VoiceInput).IsOk
            && MLPermissions.CheckPermission(MLPermission.ReadExternalStorage).IsOk
            && MLPermissions.CheckPermission(MLPermission.WriteExternalStorage).IsOk)
        {
            Initialize();
        }

    }

    private void Initialize()
    {
        try
        {
            Debug.Log(string.Format("[ML2VoiceCommands] Loading configuration file {0}", configFilePath));

            _voiceConfiguration = ScriptableObject.CreateInstance<MLVoiceIntentsConfiguration>();
            _voiceConfiguration.VoiceCommandsToAdd = new List<MLVoiceIntentsConfiguration.CustomVoiceIntents>();
            _voiceConfiguration.AllVoiceIntents = new List<MLVoiceIntentsConfiguration.JSONData>();
            _voiceConfiguration.SlotsForVoiceCommands = new List<MLVoiceIntentsConfiguration.SlotData>();

            readConfigFile(configFilePath, ref _voiceConfiguration);

            MLResult result = MLVoice.SetupVoiceIntents(_voiceConfiguration);

            if (result.IsOk)
            {
                MLVoice.OnVoiceEvent += onVoiceEvent;
            }
            else
            {
                Debug.LogError("[ML2VoiceCommands] Failed to Setup Voice Intents with result: " + result);
            }
        }
        catch (Exception e)
        {
            Debug.LogError(e.ToString());
        }

        Debug.Log("[ML2VoiceCommands] Successfully Set Up Voice Commands!");
    }

    private void onPermissionDenied(string permission)
    {
        Debug.LogError(string.Format("[ML2VoiceCommands] Permission {0} denied"));
    }

    private void onPermissionGranted(string permission)
    {
        Debug.Log(string.Format("[ML2VoiceCommands] Permission {0} granted"));
    }

    // Helper function to read configuration to add commands
    void readConfigFile(in string path, ref MLVoiceIntentsConfiguration voiceConfig)
    {
        uint count = 0;
        using (StreamReader sr = new StreamReader(path))
        {
            switch (configFileType)
            {
                case ConfigType.CSV:
                    count = readCSV(sr, ref voiceConfig);
                    break;
                default:
                    throw new Exception(new string("[ML2VoiceCommands] The selected configuration file format has not been implemented."));
            }
        }
        if (count == 0) throw new Exception("[ML2VoiceCommands] Configuration File \"{0}\" contains no voice command configuration data.");
    }

    uint readCSV(in StreamReader sr, ref MLVoiceIntentsConfiguration voiceConfig)
    {
        uint count = 0;
        uint idStart = 900;
        while (!sr.EndOfStream)
        {
            string line = sr.ReadLine();
            Debug.Log(string.Format("[ML2VoiceCommands] Read voice command configuration for \"{0}\" with ID {1}", line, count+idStart));
            MLVoiceIntentsConfiguration.CustomVoiceIntents newIntent;
            newIntent.Value = line;
            newIntent.Id = idStart+count;
            voiceConfig.VoiceCommandsToAdd.Add(newIntent);
            ++count;
        }
        return count;
    }

    // Callback for voice events triggers an event that sends text to the HUD
    private void onVoiceEvent(in bool wasSuccessful, in MLVoice.IntentEvent voiceEvent)
    {
        Debug.Log(string.Format("[ML2VoiceCommands] Voice Command Recognized\"{0}\"", voiceEvent.EventName));
    }

    private void OnDestroy()
    {
        MLVoice.Stop();
        MLVoice.OnVoiceEvent -= onVoiceEvent;

        callbacks.OnPermissionGranted -= onPermissionGranted;
        callbacks.OnPermissionDenied -= onPermissionDenied;
        callbacks.OnPermissionDeniedAndDontAskAgain -= onPermissionDenied;
    }
}

The script makes it past the exception handler, all the way to the success debug message on Line 99. Here is the log output to that point:

2023-05-30 19:58:58.381;10110;29253;29285;Info;Unity;[ML2VoiceCommands] Read voice command configuration for "potato" with ID 900;
2023-05-30 19:58:58.381;10110;29253;29285;Info;Unity;[ML2VoiceCommands] Read voice command configuration for "tomato" with ID 901;
2023-05-30 19:58:58.381;10110;29253;29285;Info;Unity;[ML2VoiceCommands] Read voice command configuration for "french roast" with ID 902;
2023-05-30 19:58:58.381;10110;29253;29285;Info;Unity;[ML2VoiceCommands] Read voice command configuration for "ground beef" with ID 903;
2023-05-30 19:58:58.381;10110;29253;29285;Info;Unity;[ML2VoiceCommands] Read voice command configuration for "hello world | goodbye world" with ID 904;

When I say a trigger word, the app briefly freezes, the program crashes without warning. It seems to successfully detect the word, as indicated by the following log entries:

2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:116: [nluEvent]----- [SensoryIntentSessionType 2] Remaining NLU slot count: 0;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:122: [nluEvent]----- [SensoryIntentSessionType 2] 1st NLU cb: current intent has total 0 slots;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:141: [nluEvent]----- [SensoryIntentSessionType 2] Remaining NLU slot count: 0, name = UnityApp_VoiceIntent_ID900, value = potato;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:146: [nluEvent]----- [SensoryIntentSessionType 2] NLU parsed intent Name = UnityApp_VoiceIntent_ID900, transcript(also contains implicit slot names) = potato;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:98: [setVIParceableIntentFields]-----  sys intent id = 33, sys intent str = ML_APP_INTENT, app intentName = UnityApp_VoiceIntent_ID900, app_intent_id = 900;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:205: [nluEvent]----- [SensoryIntentSessionType 2] [Debug!!!] Got an intent [sys intent = ML_APP_INTENT , app intent id = 900] to be arbitrated internally first!!!!;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:210: [nluEvent]----- [SensoryIntentSessionType 2]------ an intent [sys intent = ML_APP_INTENT , app intent id = 900] has been written into the internal msg queue!!!!;
2023-05-30 19:59:15.955;4034;3460;6204;Info;VoiceService;offline_intent_iface.cpp:638: [arbitrationThread]----- To arbitrate a RESULT->INTENT [sys intent ML_APP_INTENT, app intent id: 900] from intentsession type [2] in the internal msg queue!;
2023-05-30 19:59:15.955;4034;3460;6207;Info;VoiceService;sensory_iface.cpp:80: [SensoryIntentSessionType 2] showAlignment: strat_ts= 15660, end_ts= 16140 score=0.00, phrase=potato;
2023-05-30 19:59:15.959;4051;27231;27235;Info;cvip-ch;May 30 19:59:13 0000002932304525896 ARM.05 00934 me_38a8bcb4da05 [algo/arm/src/wr_arm_blocks.cpp:00352]:I:: :wr: Reconstructing depth frame 6341, 1365387554us;
2023-05-30 19:59:15.976;4034;3460;6204;Info;VoiceService;offline_intent_iface.cpp:671: [arbitrationThread]----- app wuw disabled while no sys wuw enabled or detected, got a valid app intent id [900] in the internal msg queue!;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_timer.cpp:68: Stopping timer;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1647: current intent (sys intent: 33 , app intent: 900) from session_type 2 is to be broadcasted;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_timer.cpp:68: Stopping timer;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_timer.cpp:45: Starting timer;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1320: [BroadcastResult] ----- to broadcast this VoiceResultParceable:;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1704: ------------ current VoiceResultParceable:;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1705: --- text: potato;
2023-05-30 19:59:15.976;4034;3460;6205;Info;VoiceService;local_intent_client.cpp:154: Received Intent 33;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1706: --- confidence = 1.000000, asr_confidence = 1.000000;
2023-05-30 19:59:15.976;4034;3460;6205;Info;VoiceService;local_intent_client.cpp:178: Local intent client cannot handle this result;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1707: --- is_final = 1, result_code = 0;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1710: --- stream_mode = 1, context = 0;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1716: --- system intent = 33, intent_app = 900;
2023-05-30 19:59:15.976;4034;3460;6208;Info;VoiceService;voice_service.cpp:1724: --- no_intent_rc = 0;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:199: ****************************************************;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:200: Mode: 1;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:201: System Intent: 33;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:202: App Intent: 900;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:203: No Intent Reason Code: 0;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:204: Confidence: 1.000000;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:205: Text: potato;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:206: Final: 1;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:207: Result code: 0;
2023-05-30 19:59:15.976;4046;5359;6170;Info;NovaInputService;voice_device.cpp:214: ****************************************************;

On another run, I was able to get it to spit out a whole host of other errors, after encountering the following:

2023-05-30 21:01:29.619;10110;6869;6887;Error;libc++abi;terminating with uncaught exception of type Il2CppExceptionWrapper;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;Version '2022.2.0b16 (3c3b3e6cd1d7)', Build type 'Release', Scripting Backend 'il2cpp', CPU 'x86_64';
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;Build fingerprint: 'ml/demophon_aosp/demophon:10/B3E.221117.04-R.028/30:user/release-keys';
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;Revision: '0';
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;ABI: 'x86_64';
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;Timestamp: 2023-05-30 21:01:29.620594986+0000;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;pid: 6869, tid: 6887, name: Binder:6869_3  >>> com.DefaultCompany.ML2VoiceCommands <<<;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;uid: 10110;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr --------;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;rax 0000000000000000  rbx 0000000000001ad5  rcx 000070309ceef288  rdx 0000000000000006;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;r8  0000702fc701ac00  r9  0000000000000000  r10 0000702fc701abc0  r11 0000000000000246;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;r12 0000702fc701ae00  r13 0000003000000008  r14 0000702fc701ac58  r15 0000000000001ae7;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;rdi 0000000000001ad5  rsi 0000000000001ae7;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;rbp 0000702fc701acc0  rsp 0000702fc701abb8  rip 000070309ceef288;
2023-05-30 21:01:29.620;10110;6869;6887;Error;CRASH;;

Some details:
Unity Editor version: 2022.2.0b16.112.5806
ML2 OS version: 1.1.0-dev2
MLSDK version: 1.3.0-dev1
Host OS: Windows 10

I've attached a whole host of log files in which the program crashed, some filtered on the PID of the running app. Hoping to solve this issue soon. Thank you!
device_Log_20230530_1522_49.txt (38.4 KB)
device_Log_20230530_1553_10.txt (1.1 MB)
device_Log_20230530_1653_25.txt (38.4 KB)
device_Log_20230530_1654_31.txt (36.8 KB)
device_Log_20230530_1713_40.txt (72.4 KB)

etucker · May 30, 2023, 10:10pm

Hi @audeo,

Welcome to the forums and thank you for bringing this to our attention. I will consult our devs and report back once we have a lead on this issue.

Thanks,

El

etucker · May 31, 2023, 2:39pm

Hi. Could you please print the MLVoiceIntentsConfiguration scriptable object using the function called GetJSONString() in your logs and send us a screenshot?

audeo · May 31, 2023, 3:47pm

Hi, the contents of the voice config are as follows:

{"app_intents":[{"name":"UnityApp_VoiceIntent_ID900","id":"900","value":"potato"},{"name":"UnityApp_VoiceIntent_ID901","id":"901","value":"tomato"},{"name":"UnityApp_VoiceIntent_ID902","id":"902","value":"french roast"},{"name":"UnityApp_VoiceIntent_ID903","id":"903","value":"ground beef"},{"name":"UnityApp_VoiceIntent_ID904","id":"904","value":"hello world | goodbye world"}],"sys_intent_list":{"name":[]},"app_slots":[]}

audeo · May 31, 2023, 3:52pm

For further information, here is my scene setup:

The script in my original post is attached to an empty game object named "Voice Intents."

audeo · May 31, 2023, 6:17pm

It appears the issue I had was caused by a mismatch in the OS and the SDK version. I am using the MLSDK v1.3.0-dev1, and the headset was on OS v1.1.0-dev2. I updated the headset to the latest OS, v1.3.0-dev1 (which matches the SDK version number) and now it works!!!

Topic		Replies	Views
Voice Intents Not Working with MLSDK Unity Development unity , voice-intents	10	40	October 4, 2024
Voice Intents Starting but OnVoiceEvent Not Being Called Input/Interaction unity	1	203	September 14, 2023
Voice Intents not working Unity Development	2	223	October 3, 2022
Non-recoverable voice command failure due to "app intents is larger than the allowed max 100" MRTK Development	4	21	January 17, 2025
Voice intents bug when multiple Data Slots are used in one Voice Command Unity Development development , unity	6	220	August 4, 2023

Runtime Voice Intents Crash Unity Application on Command Recognition

Related topics