Camera capture timestamp delta

Unity Development
, ,
9

Thanks @kbabilinski for your notes and working code.

The code you posted is for video capture, but we're using still image capture for better control of autofocus, white balance, et cetera. Also, we need to capture at 4096x3072.

I've posted our code below, which in cross-testing I can confirm works using Unity framework [v1.8.0] but doesn't work on [v2.0.0]. In both Unity projects, we're using MLSDK 1.5.0 and Unity Editor 2022.3.13f1.

Specifically, with [v1.8.0] we are able to call the image prep, save a full-resolution image, and retrieve both intrinsics and extrinsics as text files. When running Unity with ML API [v2.0.0], we receive a NULL image and see the following error in logcat:

Error Camera-Device endConfigure fail Status(-8, EX_SERVICE_SPECIFIC): '3: endConfigure:739: Camera 0: Unsupported set of inputs/outputs provided'

Is there a change in handling ML Cam in the later release?

  1. Strangely, when the capture fails, we still get a zero-byte image file. I added logging, which shows size and data length are zero, but Data returns 'Data available':

Image capture complete.
Number of planes: 1
Planes[0].Size: 0
Planes[0].Data length: 0 bytes
Planes[0].Data: Data available
Image saved to /storage/emulated/0/Android/data/.../236186942000.jpg

  1. In a previous post you were able to crash your ML2 (as I was) by setting MLCamera.CaptureFrameRate._15FPS. That thread suggests this is fixed, but I can confirm it's not: setting 15FPS in the code below will crash the ML2 and require both the device and system to be cycled to recover(!)

  2. Note that we're saving as .jpg, which in another of your posts you note shouldn't work: "The Magic Leap 2's CV Camera stream does not support JPG image capture at this time."

Here's our working code (with[v1.8.0], that is):

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MagicLeap.Core;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.XR.MagicLeap;

//############################
//  ___                 _   
// | . \ ___  _ _  ___ | |__
// |   // ._>| | |/ . \| / /
// |_\_\\___.|__/ \___/|_\_\
//                         
// Magic Leap Revok PoC
// (c) 2024 Plexus Software LLC
//
// This script exports ML2 camera data: images, instrinsics and extrinsics.

public class RGBCamCapture : MonoBehaviour
{
    private static RGBCamCapture _instance;
    public static RGBCamCapture Instance
    {
        get
        {
            if (_instance == null)
            {
                var existingInstance = FindObjectOfType<RGBCamCapture>();
                if (existingInstance != null)
                {
                    _instance = existingInstance;
                }
                else
                {
                    GameObject newGameObject = new GameObject("RGBCamCapture");
                    _instance = newGameObject.AddComponent<RGBCamCapture>();
                }
            }
            return _instance;
        }
    }

    private bool isCameraConnected = false;
    private MLCamera colorCamera;
    private bool cameraDeviceAvailable;
    private readonly MLPermissions.Callbacks permissionCallbacks = new MLPermissions.Callbacks();

    private void Awake()
    {
        if (_instance != null && _instance != this)
        {
            Destroy(gameObject);
            return;
        }
        _instance = this;
        DontDestroyOnLoad(gameObject);

        permissionCallbacks.OnPermissionGranted += OnPermissionGranted;
        permissionCallbacks.OnPermissionDenied += OnPermissionDenied;
        permissionCallbacks.OnPermissionDeniedAndDontAskAgain += OnPermissionDenied;
    }

    void Start()
    {
        MLResult result = MLPermissions.RequestPermission(MLPermission.Camera, permissionCallbacks);

        if (!result.IsOk)
        {
            Debug.LogErrorFormat("Error: RGBCamCapture failed to get requested permissions, disabling script. Reason: {0}", result);
            enabled = false;
        }
    }

    void OnDisable()
    {
        permissionCallbacks.OnPermissionGranted -= OnPermissionGranted;
        permissionCallbacks.OnPermissionDenied -= OnPermissionDenied;
        permissionCallbacks.OnPermissionDeniedAndDontAskAgain -= OnPermissionDenied;

        if (colorCamera != null && isCameraConnected)
        {
            DisableMLCamera();
        }
    }

    private void OnPermissionDenied(string permission)
    {
        LogError($"{permission} denied, camera functionality most likley won't work.");
    }

    private void OnPermissionGranted(string permission)
    {
        StartCoroutine(EnableMLCamera());
    }

    private IEnumerator EnableMLCamera()
    {
        while (!cameraDeviceAvailable)
        {
            MLResult result = MLCamera.GetDeviceAvailabilityStatus(MLCamera.Identifier.Main, out cameraDeviceAvailable);
            if (!(result.IsOk && cameraDeviceAvailable))
            {
                yield return new WaitForSeconds(1.0f);
            }
        }

        Log("Camera device available.");
        ConnectCamera();
    }

    private async void ConnectCamera()
    {
        MLCamera.ConnectContext context = MLCamera.ConnectContext.Create();
        context.EnableVideoStabilization = false;
        context.Flags = MLCameraBase.ConnectFlag.CamOnly;

        try
        {
            colorCamera = await MLCamera.CreateAndConnectAsync(context);
            if (colorCamera != null)
            {
                colorCamera.OnRawImageAvailable += OnCaptureRawImageComplete;
                isCameraConnected = true;
                Log("Camera device connected.");
                // Optionally, configure and prepare capture here or elsewhere after connection
                ConfigureAndPrepareCapture();
            }
            else
            {
                LogError("Failed to connect MLCamera: colorCamera is null.");
            }
        }
        catch (System.Exception e)
        {
            LogError($"Failed to connect MLCamera: {e.Message}");
        }
    }

    private void ConfigureAndPrepareCapture()
    {
        // Assuming configuration setup is similar to the async example, without the async/await pattern
        // We set 30FPS capture to avoid problems: https://forum.magicleap.cloud/t/camera-capture-in-unity/2718/13
        MLCamera.CaptureConfig captureConfig = new MLCamera.CaptureConfig()
        {
            StreamConfigs = new[]
            {
                new MLCamera.CaptureStreamConfig()
                {
                    OutputFormat = MLCamera.OutputFormat.JPEG,
                    CaptureType = MLCamera.CaptureType.Image,
                    Width = 1920,
                    Height = 1080
                }
            },
            CaptureFrameRate = MLCamera.CaptureFrameRate._30FPS
        };

        MLResult result = colorCamera.PrepareCapture(captureConfig, out MLCamera.Metadata _);
        if (!result.IsOk)
        {
            LogError("Failed to prepare camera for capture.");
        }
    }

    public async void CaptureImage()
    {
        if (!isCameraConnected || colorCamera == null)
        {
            LogError("[Revok] Camera is not connected - cannot capture image.");
            return;
        }

        Log("[Revok] Preparing for image capture with AE/AWB adjustments.");

        // Attempt to adjust auto-exposure and white balance asynchronously before capture.
        var aeawbResult = await colorCamera.PreCaptureAEAWBAsync();
        if (!aeawbResult.IsOk)
        {
            LogError("[Revok] PreCaptureAEAWB failed.");
            return;
        }

        Log("[Revok] AE/AWB adjustment complete. Capturing image...");

        // Capture the image asynchronously.
        var captureResult = await colorCamera.CaptureImageAsync();
        if (!captureResult.IsOk)
        {
            LogError("[Revok] Image capture failed.");
        }
        else
        {
            Log("[Revok] Image capture successful.");
        }
    }

    private void DisableMLCamera()
    {
        if (colorCamera != null)
        {
            colorCamera.Disconnect();
            isCameraConnected = false;
        }
    }

    public async Task CaptureImageAsync()
    {
        if (isCameraConnected && colorCamera != null)
        {
            Log("[Revok] Preparing for image capture with AE/AWB adjustments.");

            var aeAwbResult = await colorCamera.PreCaptureAEAWBAsync();
            if (!aeAwbResult.IsOk)
            {
                LogError("[Revok] PreCaptureAEAWB failed.");
                return;
            }

            Log("[Revok] AE/AWB adjustment complete. Capturing image...");

            var captureResult = await colorCamera.CaptureImageAsync();
            if (!captureResult.IsOk)
            {
                LogError("[Revok] Image capture failed.");
            }
            else
            {
                Log("[Revok] Image capture successful.");
            }
        }
        else
        {
            LogError("[Revok] Camera is not connected - cannot capture image.");
        }
    }

    private void OnCaptureRawImageComplete(MLCamera.CameraOutput capturedImage, MLCamera.ResultExtras resultExtras, MLCamera.Metadata metadataHandle)
    {
        Log("Image capture complete.");

        // Adjust the directory path to include 'capture'
        string directoryPath = Path.Combine(Application.persistentDataPath, "capture");
        // Check if the 'capture' directory exists, create it if needed
        if (!Directory.Exists(directoryPath))
        {
            Directory.CreateDirectory(directoryPath);
        }

        // Adjust file paths to use the updated directoryPath
        string fileName = $"{resultExtras.VCamTimestamp}.txt";
        string filePath = Path.Combine(directoryPath, fileName);

        string imagePath = Path.Combine(directoryPath, $"{resultExtras.VCamTimestamp}.png");
        File.WriteAllBytes(imagePath, capturedImage.Planes[0].Data);
        Log($"Image saved to {imagePath}");

        // Attempt to get the camera pose
        if (resultExtras.VCamTimestamp > 0)
        {
            Matrix4x4 outTransform;
            MLResult poseResult = MLCVCamera.GetFramePose(resultExtras.VCamTimestamp, out outTransform);

            if (poseResult == MLResult.Code.Ok)
            {
                // Prepare the content for the file
                string fileContent = "intrinsics\n";
                fileContent += resultExtras.Intrinsics.HasValue ? resultExtras.Intrinsics.Value.ToString() : "Not Available";
                fileContent += "\nextrinsics\n";
                fileContent += outTransform.ToString();

                // Write to the file
                File.WriteAllText(filePath, fileContent);
                Log($"Data saved to {filePath}");
            }
            else
            {
                Log($"Failed to obtain camera pose: {poseResult}");
            }
        }
        else
        {
            Log("VCamTimestamp is not valid.");
        }
    }

    private void Log(string message)
    {
        Debug.Log("[Revok] " + message);
    }

    private void LogError(string message)
    {
        Debug.LogError("[Revok] " + message);
    }

}