Camera capture in Unity

Unity Development
, ,
4

I've added more checking and more explicit stream configuration, below. Logcat is showing two streams as before:

Info ml_camera_client  Capture device state = Idle numStreamConfigs = 1 
Info ml_camera_client  Stream 0 type = Image resolution = 1080P state = Configured 
Info ml_camera_client Stream 1 type = Image resiolution =  state = Invalid

It appears from the above the stream 0 is configured for still images; I am trying to specify the stream via [0] or [1] here:
MLCamera.StreamCapability selected = streamCapabilities[0];
and
selectedStreamCapability = streamCapabilityOptions[0];

In either case, captureCamera.CaptureImage() fails.

Full code follows:

using UnityEngine;
using UnityEngine.XR.MagicLeap;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine.InputSystem;
using System;
using static UnityEngine.XR.MagicLeap.MLCameraBase;
using System.Collections.Generic;
using UnityEngine.Playables;

public class SimpleCameraCapture : MonoBehaviour
{
[SerializeField] private int width = 640;
[SerializeField] private int height = 480;
[SerializeField] private MLCamera.CaptureFrameRate frameRate = MLCamera.CaptureFrameRate._15FPS;
[SerializeField] private List streamCapabilityOptions; // To hold the dropdown options
[SerializeField] private string selectedStreamCapability; // To hold the selected dropdown option

private MLCamera captureCamera;
private MLCamera.CaptureStreamConfig captureStreamConfig; // Declare as member variable
private bool cameraDeviceAvailable = false;
private List<MLCamera.StreamCapability> streamCapabilities;

private void Start()
{
    MLPermissions.Callbacks permissionCallbacks = new MLPermissions.Callbacks();
    permissionCallbacks.OnPermissionGranted += OnPermissionGranted;
    permissionCallbacks.OnPermissionDenied += OnPermissionDenied;
    permissionCallbacks.OnPermissionDeniedAndDontAskAgain += OnPermissionDeniedAndDontAskAgain;

    MLPermissions.RequestPermission(MLPermission.Camera, permissionCallbacks);
}

private void OnPermissionGranted(string permission)
{
    StartCoroutine(EnableMLCamera());
}

private void OnPermissionDenied(string permission)
{
    Debug.Log("Permission denied: " + permission);
}

private void OnPermissionDeniedAndDontAskAgain(string permission)
{
    Debug.Log("Permission denied and don't ask again: " + permission);
}

private IEnumerator EnableMLCamera()
{
    while (!cameraDeviceAvailable)
    {
        MLResult result = MLCamera.GetDeviceAvailabilityStatus(MLCamera.Identifier.Main, out cameraDeviceAvailable);
        if (!(result.IsOk && cameraDeviceAvailable))
        {
            yield return new WaitForSeconds(1.0f);
        }
    }

    Debug.Log("Camera device available");
    ConnectCamera();
}

/// Update the streamCapabilityOptions based on the available streamCapabilities
private void UpdateStreamCapabilityOptions()
{
    streamCapabilityOptions = new List<string>();
    foreach (var cap in streamCapabilities)
    {
        streamCapabilityOptions.Add($"{cap.Width}x{cap.Height}");
    }
}

private void ConnectCamera()
{
    MLCamera.ConnectContext context = MLCamera.ConnectContext.Create();
    context.Flags = MLCamera.ConnectFlag.CamOnly;
    context.EnableVideoStabilization = false;

    captureCamera = MLCamera.CreateAndConnect(context);

    if (captureCamera != null)
    {
        Debug.Log("ML2 RGB Camera connected!");

        // Initialize stream capabilities and dropdown options
        if (!GetImageStreamCapabilities())
        {
            Debug.LogError("Failed to get image stream capabilities.");
            // Handle this error appropriately
        }
        else
        {
            if (streamCapabilityOptions.Count > 0)
            {
                // Update the selectedStreamCapability based on the available options
                // 0 or 1
                selectedStreamCapability = streamCapabilityOptions[0];
            }
        }
    }
}

private void OnCaptureRawImageComplete(MLCamera.CameraOutput output, MLCamera.ResultExtras resultExtras, MLCamera.Metadata metadataHandle)
{
    Debug.Log("RGB image callback!");
}

/// Gets the Image stream capabilities
/// True if MLCamera returned at least one stream capability
private bool GetImageStreamCapabilities()
{
    var result =
        captureCamera.GetStreamCapabilities(out MLCamera.StreamCapabilitiesInfo[] streamCapabilitiesInfo);

    if (!result.IsOk)
    {
        Debug.Log("Could not get Stream capabilities Info.");
        return false;
    }

    streamCapabilities = new List<MLCamera.StreamCapability>();
    streamCapabilityOptions = new List<string>(); // Initialize the list

    for (int i = 0; i < streamCapabilitiesInfo.Length; i++)
    {
        foreach (var streamCap in streamCapabilitiesInfo[i].StreamCapabilities)
        {
            streamCapabilities.Add(streamCap);
            streamCapabilityOptions.Add($"{streamCap.Width}x{streamCap.Height}"); // Populate the options
        }
    }

    // Set a default value for the selected capability if possible
    if (streamCapabilityOptions.Count > 0)
    {
        Debug.Log("Selected stream capability set to [1] -------------------------------");
        selectedStreamCapability = streamCapabilityOptions[1];
    }

    return streamCapabilities.Count > 0;
}

private MLCamera.StreamCapability GetStreamCapability()
{
    // Check if streamCapabilities has at least two elements
    // 1 or 2
    if (streamCapabilities.Count < 2)
    {
        Debug.LogError("Insufficient stream capabilities.");
    }

    //  // Default to Stream 0 or 1
    MLCamera.StreamCapability selected = streamCapabilities[1];
    foreach (var streamCapability in streamCapabilities)
    {
        if ($"{streamCapability.Width}x{streamCapability.Height}" == selectedStreamCapability)
        {
            selected = streamCapability;
            break;
        }
    }

    Debug.Log($"Returning stream capability for still images: Width = {selected.Width}, Height = {selected.Height}");

    return selected;
}

void Update()
{
    if (Keyboard.current.spaceKey.wasPressedThisFrame && captureCamera != null)
    {
        // Check for initialized stream capabilities
        if (streamCapabilities == null || !streamCapabilities.Any())
        {
            // Stream capabilities not initialized
            Debug.Log("Stream capabilities not initialized ---------------------------------");
            return;
        }

        // Initialize CaptureConfig
        Debug.Log("Stream capabilities initialized ---------------------------------");

        // Get the stream capability for Stream 1 (still images)
        MLCamera.StreamCapability selectedCapability = GetStreamCapability();

        int captureWidth = (selectedCapability.Width > 0) ? selectedCapability.Width : width;
        int captureHeight = (selectedCapability.Height > 0) ? selectedCapability.Height : height;

        MLCamera.CaptureConfig captureConfig = new MLCamera.CaptureConfig
        {
            CaptureFrameRate = frameRate,
            StreamConfigs = new MLCamera.CaptureStreamConfig[]
            {
        new MLCamera.CaptureStreamConfig()
        {
            OutputFormat = MLCamera.OutputFormat.JPEG,
            CaptureType = MLCamera.CaptureType.Image,
            Width = captureWidth,  // Use either the stream's width or a default
            Height = captureHeight  // Use either the stream's height or a default
        }
            }
        };

        // Prepare for capture
        Debug.Log("Prepare Capture ---------------------------------");
        MLResult result = captureCamera.PrepareCapture(captureConfig, out MLCamera.Metadata _);

        if (MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.PrepareCapture)))
        {
            // Trigger auto-exposure and auto-white-balance convergence
            Debug.Log("Trigger auto-exposure and auto-white-balance convergence ---------------------------------");
            captureCamera.PreCaptureAEAWB();
        }
        else
        {
            Debug.Log("pre capture failed ---------------------------------");
            // Failed to prepare the camera
            return;
        }

        // Capture the image
        Debug.Log("Capture RGB ---------------------------------");
        result = captureCamera.CaptureImage(1);
        if (!result.IsOk)
        {
            // Image capture failed
            return;
        }
    }
}

void OnDestroy()
{
    if (captureCamera != null)
    {
        captureCamera.OnRawImageAvailable -= OnCaptureRawImageComplete;
        captureCamera.Disconnect();
    }
}

}

2 Likes