Hi @usman.bashir
thank you for your patience, I spoke with some other engineers and we think this should work for you
// %BANNER_BEGIN%
// ---------------------------------------------------------------------
// %COPYRIGHT_BEGIN%
// Copyright (c) (2019-2022) Magic Leap, Inc. All Rights Reserved.
// Use of this file is governed by the Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2
// Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein.
// %COPYRIGHT_END%
// ---------------------------------------------------------------------
// %BANNER_END%
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using MagicLeap.Core;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.XR.MagicLeap;
namespace MagicLeap.Examples
{
/// <summary>
/// This class handles video recording and image capturing based on controller
/// input.
/// </summary>
public class CameraCaptureExample : MonoBehaviour
{
[SerializeField, Tooltip("The text used to display status information for the example.")]
private Text statusText = null;
[SerializeField, Tooltip("Refrence to the Raw Video Capture Visualizer gameobject for YUV frames")]
private CameraCaptureVisualizer cameraCaptureVisualizer = null;
[SerializeField, Tooltip("Reference to media player behavior used in camera capture playback")]
private MLMediaPlayerBehavior mediaPlayerBehavior;
[Header("UI Objects")]
[SerializeField, Tooltip("Button that starts the Capture")]
private Button captureButton;
[SerializeField, Tooltip("Button that connects to MLCamera")]
private Button connectButton;
[SerializeField, Tooltip("Button that disconnect from MLCamera")]
private Button disconnectButton;
private Coroutine recordingRoutine;
private bool IsCameraConnected => captureCamera != null && captureCamera.ConnectionEstablished;
private MLCamera captureCamera;
private readonly CameraRecorder cameraRecorder = new CameraRecorder();
private string recordedFilePath;
private bool cameraDeviceAvailable;
private bool isCapturingVideo = false;
private bool isCapturingPreview = false;
private const string validFileFormat = ".mp4";
private MLCamera.ConnectFlag ConnectFlag => MLCamera.ConnectFlag.CamOnly;
private MLCamera.CaptureType CaptureType => MLCamera.CaptureType.Video;
private MLCamera.MRQuality MRQuality => MLCamera.MRQuality._1440x1080;
private MLCamera.CaptureFrameRate FrameRate => MLCamera.CaptureFrameRate._30FPS;
private MLCamera.OutputFormat OutputFormat => MLCamera.OutputFormat.YUV_420_888;
private bool RecordToFile = true;
private readonly MLPermissions.Callbacks permissionCallbacks = new MLPermissions.Callbacks();
private bool skipFrame = false;
private void Awake()
{
permissionCallbacks.OnPermissionGranted += OnPermissionGranted;
permissionCallbacks.OnPermissionDenied += OnPermissionDenied;
permissionCallbacks.OnPermissionDeniedAndDontAskAgain += OnPermissionDenied;
captureButton.onClick.AddListener(OnCaptureButtonClicked);
connectButton.onClick.AddListener(ConnectCamera);
disconnectButton.onClick.AddListener(DisconnectCamera);
RefreshUI();
}
private void Start()
{
MLPermissions.RequestPermission(MLPermission.Camera, permissionCallbacks);
MLPermissions.RequestPermission(MLPermission.RecordAudio, permissionCallbacks);
TryEnableMLCamera();
}
/// <summary>
/// Stop the camera, unregister callbacks.
/// </summary>
void OnDisable()
{
DisconnectCamera();
}
/// <summary>
/// Handle Camera connection if application is paused.
/// </summary>
private void OnApplicationPause(bool isPaused)
{
if (isPaused && IsCameraConnected)
{
StopCoroutine(recordingRoutine);
StopVideoCapture();
DisconnectCamera();
}
else
{
mediaPlayerBehavior.Reset();
mediaPlayerBehavior.gameObject.SetActive(false);
}
}
/// <summary>
/// Display permission error if necessary or update status text.
/// </summary>
private void Update()
{
UpdateStatusText();
}
private void OnDestroy()
{
permissionCallbacks.OnPermissionGranted -= OnPermissionGranted;
permissionCallbacks.OnPermissionDenied -= OnPermissionDenied;
permissionCallbacks.OnPermissionDeniedAndDontAskAgain -= OnPermissionDenied;
}
private void TryEnableMLCamera()
{
if (!MLPermissions.CheckPermission(MLPermission.Camera).IsOk)
return;
StartCoroutine(EnableMLCamera());
}
/// <summary>
/// Connects the MLCamera component and instantiates a new instance
/// if it was never created.
/// </summary>
private IEnumerator EnableMLCamera()
{
while (!cameraDeviceAvailable)
{
MLResult result =
MLCamera.GetDeviceAvailabilityStatus(MLCamera.Identifier.Main, out cameraDeviceAvailable);
if (!(result.IsOk && cameraDeviceAvailable))
{
// Wait until camera device is available
yield return new WaitForSeconds(1.0f);
}
}
Debug.Log("Camera device available");
}
/// <summary>
/// Connects to the MLCamera.
/// </summary>
private void ConnectCamera()
{
MLCamera.ConnectContext context = MLCamera.ConnectContext.Create();
context.Flags = ConnectFlag;
context.EnableVideoStabilization = true;
if (context.Flags != MLCamera.ConnectFlag.CamOnly)
{
context.MixedRealityConnectInfo = MLCamera.MRConnectInfo.Create();
context.MixedRealityConnectInfo.MRQuality = MRQuality;
context.MixedRealityConnectInfo.MRBlendType = MLCamera.MRBlendType.Additive;
context.MixedRealityConnectInfo.FrameRate = FrameRate;
}
captureCamera = MLCamera.CreateAndConnect(context);
if (captureCamera != null)
{
Debug.Log("Camera device connected");
captureCamera.OnRawVideoFrameAvailable += OnCaptureRawVideoFrameAvailable;
}
RefreshUI();
}
/// <summary>
/// Disconnects the camera.
/// </summary>
private void DisconnectCamera()
{
if (captureCamera == null || !IsCameraConnected)
{
// Note that some APIs like MLCameraInit() can be called before MLCameraConnect()
// is called. This is to make sure all is cleaned up if CameraConnect is not called
MLCamera.Uninitialize();
return;
}
captureCamera.OnRawVideoFrameAvailable -= OnCaptureRawVideoFrameAvailable;
// media player not supported in Magic Leap App Simulator
#if !UNITY_EDITOR
mediaPlayerBehavior.MediaPlayer.OnPrepared -= MediaPlayerOnOnPrepared;
mediaPlayerBehavior.MediaPlayer.OnCompletion -= MediaPlayerOnCompletion;
#endif
captureCamera.Disconnect();
RefreshUI();
}
/// <summary>
/// Gets currently selected StreamCapability
/// </summary>
private MLCamera.StreamCapability GetStreamCapability(MLCameraBase.CaptureType captureType)
{
MLCameraBase.StreamCapability[] targetStreamCapabilities = MLCamera.GetImageStreamCapabilitiesForCamera(captureCamera, captureType);
if (MLCamera.TryGetBestFitStreamCapabilityFromCollection(targetStreamCapabilities, 1080, 120,
captureType, out MLCameraBase.StreamCapability capability))
{
return capability;
}
return targetStreamCapabilities[0];
}
/// <summary>
/// Capture Button Clicked.
/// </summary>
private void OnCaptureButtonClicked()
{
if (isCapturingVideo || isCapturingPreview)
return;
if (GetStreamCapability(CaptureType).CaptureType == MLCamera.CaptureType.Video ||
GetStreamCapability(CaptureType).CaptureType == MLCamera.CaptureType.Preview)
{
StartVideoCapture();
recordingRoutine = StartCoroutine(StopVideo());
}
RefreshUI();
}
private IEnumerator StopVideo()
{
float startTimestamp = Time.realtimeSinceStartup;
while (Time.realtimeSinceStartup - startTimestamp < 10)
{
yield return null;
}
StopVideoCapture();
RefreshUI();
}
private void StartVideoCapture()
{
recordedFilePath = string.Empty;
skipFrame = false;
var result = MLPermissions.CheckPermission(MLPermission.Camera);
MLResult.DidNativeCallSucceed(result.Result, nameof(MLPermissions.RequestPermission));
if (!result.IsOk)
{
Debug.LogError($"{MLPermission.Camera} permission denied. Video will not be recorded.");
return;
}
if (RecordToFile)
StartRecording();
else
StartPreview();
}
/// <summary>
/// Captures a preview of the device's camera and displays it in front of the user.
/// If Record to File is selected then it will not show the preview.
/// </summary>
private void StartPreview()
{
MLCamera.CaptureConfig captureConfig = new MLCamera.CaptureConfig();
captureConfig.CaptureFrameRate = FrameRate;
captureConfig.StreamConfigs = new MLCamera.CaptureStreamConfig[2];
captureConfig.StreamConfigs[0] =
MLCamera.CaptureStreamConfig.Create(GetStreamCapability(CaptureType), OutputFormat);
captureConfig.StreamConfigs[1] =
MLCamera.CaptureStreamConfig.Create(GetStreamCapability(MLCameraBase.CaptureType.Preview), OutputFormat);
MLResult result = captureCamera.PrepareCapture(captureConfig, out MLCamera.Metadata _);
if (MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.PrepareCapture)))
{
captureCamera.PreCaptureAEAWB();
result = captureCamera.CaptureVideoStart();
isCapturingVideo = MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.CaptureVideoStart));
result = captureCamera.CapturePreviewStart();
isCapturingPreview = MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.CapturePreviewStart));
if (isCapturingPreview)
{
cameraCaptureVisualizer.DisplayPreviewCapture(captureCamera.PreviewTexture, RecordToFile);
}
}
}
/// <summary>
/// Stops the Video Capture.
/// </summary>
private void StopVideoCapture()
{
if (!isCapturingVideo && !isCapturingPreview)
return;
if (isCapturingVideo)
{
captureCamera.CaptureVideoStop();
}
else if (isCapturingPreview)
{
captureCamera.CapturePreviewStop();
}
cameraCaptureVisualizer.HideRenderer();
if (RecordToFile)
{
StopRecording();
DisplayPlayback();
}
isCapturingVideo = false;
isCapturingPreview = false;
}
/// <summary>
/// Starts Recording camera capture to file.
/// </summary>
private void StartRecording()
{
// media player not supported in Magic Leap App Simulator
#if !UNITY_EDITOR
mediaPlayerBehavior.MediaPlayer.OnPrepared += MediaPlayerOnOnPrepared;
mediaPlayerBehavior.MediaPlayer.OnCompletion += MediaPlayerOnCompletion;
#endif
string fileName = DateTime.Now.ToString("MM_dd_yyyy__HH_mm_ss") + validFileFormat;
recordedFilePath = System.IO.Path.Combine(Application.persistentDataPath, fileName);
CameraRecorderConfig config = CameraRecorderConfig.CreateDefault();
config.Width = GetStreamCapability(CaptureType).Width;
config.Height = GetStreamCapability(CaptureType).Height;
config.FrameRate = MapFrameRate(FrameRate);
cameraRecorder.StartRecording(recordedFilePath, config);
int MapFrameRate(MLCamera.CaptureFrameRate frameRate)
{
switch (frameRate)
{
case MLCamera.CaptureFrameRate.None: return 0;
case MLCamera.CaptureFrameRate._15FPS: return 15;
case MLCamera.CaptureFrameRate._30FPS: return 30;
case MLCamera.CaptureFrameRate._60FPS: return 60;
default: return 0;
}
}
MLCamera.CaptureConfig captureConfig = new MLCamera.CaptureConfig();
captureConfig.CaptureFrameRate = FrameRate;
captureConfig.StreamConfigs = new MLCamera.CaptureStreamConfig[2];
captureConfig.StreamConfigs[0] = MLCamera.CaptureStreamConfig.Create(GetStreamCapability(CaptureType), OutputFormat);
captureConfig.StreamConfigs[0].Surface = cameraRecorder.MediaRecorder.InputSurface;
captureConfig.StreamConfigs[1] =
MLCamera.CaptureStreamConfig.Create(GetStreamCapability(MLCameraBase.CaptureType.Preview), OutputFormat);
MLResult result = captureCamera.PrepareCapture(captureConfig, out MLCamera.Metadata _);
Debug.Log($"Recording to {OutputFormat}. Width: {GetStreamCapability(CaptureType).Width} Height: {GetStreamCapability(CaptureType).Height}");
if (MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.PrepareCapture)))
{
captureCamera.PreCaptureAEAWB();
result = captureCamera.CaptureVideoStart();
isCapturingVideo = MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.CaptureVideoStart));
if (isCapturingVideo)
{
cameraCaptureVisualizer.DisplayCapture(captureConfig.StreamConfigs[0].OutputFormat, RecordToFile);
}
result = captureCamera.CapturePreviewStart();
isCapturingPreview = MLResult.DidNativeCallSucceed(result.Result, nameof(captureCamera.CapturePreviewStart));
if (isCapturingPreview)
{
cameraCaptureVisualizer.DisplayPreviewCapture(captureCamera.PreviewTexture, false);
}
}
}
/// <summary>
/// Stops recording.
/// </summary>
private void StopRecording()
{
MLResult result = cameraRecorder.EndRecording();
if (!result.IsOk)
{
Debug.Log("Saving Recording failed, reason:" + result);
recordedFilePath = string.Empty;
}
else
{
Debug.Log("Recording saved at path: " + recordedFilePath);
}
}
/// <summary>
/// Displays recorded video.
/// </summary>
private void DisplayPlayback()
{
mediaPlayerBehavior.gameObject.SetActive(true);
mediaPlayerBehavior.source = recordedFilePath;
// media player not supported in Magic Leap App Simulator
#if !UNITY_EDITOR
mediaPlayerBehavior.PrepareMLMediaPlayer();
#endif
}
private void MediaPlayerOnOnPrepared(MLMedia.Player mediaplayer)
{
// media player not supported in Magic Leap App Simulator
#if !UNITY_EDITOR
mediaPlayerBehavior.Play();
#endif
}
private void MediaPlayerOnCompletion(MLMedia.Player mediaplayer)
{
// media player not supported in Magic Leap App Simulator
#if !UNITY_EDITOR
mediaPlayerBehavior.StopMLMediaPlayer();
#endif
mediaPlayerBehavior.gameObject.SetActive(false);
mediaPlayerBehavior.Reset();
}
/// <summary>
/// Handles the event of a new image getting captured.
/// </summary>
/// <param name="capturedFrame">Captured Frame.</param>
/// <param name="resultExtras">Result Extra.</param>
private void OnCaptureRawVideoFrameAvailable(MLCamera.CameraOutput capturedFrame, MLCamera.ResultExtras resultExtras, MLCamera.Metadata metadataHandle)
{
if (OutputFormat == MLCamera.OutputFormat.RGBA_8888 && FrameRate == MLCamera.CaptureFrameRate._30FPS && GetStreamCapability(CaptureType).Width >= 3840)
{
// cameraCaptureVisualizer cannot handle throughput of:
// 1) RGBA_8888 3840x2160 at 30 fps
// 2) RGBA_8888 4096x3072 at 30 fps
skipFrame = !skipFrame;
if (skipFrame)
{
return;
}
}
// cameraCaptureVisualizer.OnCaptureDataReceived(resultExtras, capturedFrame);
}
private void OnPermissionDenied(string permission)
{
if (permission == MLPermission.Camera)
{
MLPluginLog.Error($"{permission} denied, example won't function.");
}
else if (permission == MLPermission.RecordAudio)
{
MLPluginLog.Error($"{permission} denied, audio wont be recorded in the file.");
}
RefreshUI();
}
private void OnPermissionGranted(string permission)
{
MLPluginLog.Debug($"Granted {permission}.");
TryEnableMLCamera();
RefreshUI();
}
#region UI functions
/// <summary>
/// Updates examples status text.
/// </summary>
private void UpdateStatusText()
{
statusText.text = $"<color=#B7B7B8><b>Controller Data</b></color>\nStatus: {ControllerStatus.Text}\n";
statusText.text += $"\nCamera Available: {cameraDeviceAvailable}";
statusText.text += $"\nCamera Connected: {IsCameraConnected}";
if (!isCapturingVideo && !isCapturingPreview && !string.IsNullOrEmpty(recordedFilePath))
{
statusText.text += $"\nRecorded video file path:\n {recordedFilePath}";
}
}
/// <summary>
/// Refresh states of all dropdowns and buttons.
/// </summary>
private void RefreshUI()
{
captureButton.gameObject.SetActive(IsCameraConnected);
connectButton.gameObject.SetActive(!IsCameraConnected);
disconnectButton.gameObject.SetActive(IsCameraConnected);
}
#endregion
}
}
I hope that helps,
-Sidney