using System; using Unity.Collections; using Unity.XR.CoreUtils; using UnityEngine.Rendering; using UnityEngine.XR.ARFoundation.InternalUtils; using UnityEngine.XR.ARSubsystems; namespace UnityEngine.XR.Simulation { /// /// Simulation implementation of /// [`XRCameraSubsystem`](xref:UnityEngine.XR.ARSubsystems.XRCameraSubsystem). /// public sealed class SimulationCameraSubsystem : XRCameraSubsystem { internal const string k_SubsystemId = "XRSimulation-Camera"; /// /// The name for the shader for rendering the camera texture. /// /// /// The name for the shader for rendering the camera texture. /// const string k_BackgroundShaderName = "Unlit/Simulation Background Simple"; /// /// The shader property name for the simple RGB component of the camera video frame. /// /// /// The shader property name for the simple RGB component of the camera video frame. /// internal const string k_TextureSinglePropertyName = "_TextureSingle"; class SimulationProvider : Provider { CameraTextureFrameEventArgs m_CameraTextureFrameEventArgs; CameraTextureProvider m_CameraTextureProvider; Camera m_Camera; Material m_CameraMaterial; XRCameraConfiguration m_XRCameraConfiguration; XRCameraIntrinsics m_XRCameraIntrinsics; double m_LastFrameTimestamp = 0; XRSupportedCameraBackgroundRenderingMode m_RequestedBackgroundRenderingMode = XRSupportedCameraBackgroundRenderingMode.BeforeOpaques; Feature m_RequestedLightEstimation = Feature.None; XRCameraTorchMode m_RequestedTorchMode = XRCameraTorchMode.Off; XRCameraTorchMode m_CurrentTorchMode = XRCameraTorchMode.Off; SimulatedLight m_MainLight; Light m_CameraTorch; SimulatedExifData m_ExifData; public override XRCpuImage.Api cpuImageApi => SimulationXRCpuImageApi.instance; public override Feature currentCamera => Feature.WorldFacingCamera; public override XRCameraConfiguration? currentConfiguration { get => m_XRCameraConfiguration; set { // Currently assuming any not null configuration is valid for simulation if (value == null) throw new ArgumentNullException("value", "cannot set the camera configuration to null"); m_XRCameraConfiguration = (XRCameraConfiguration)value; } } public override Feature currentLightEstimation => base.currentLightEstimation; public override Feature requestedLightEstimation { get => m_RequestedLightEstimation; set => m_RequestedLightEstimation = value; } public override Material cameraMaterial => m_CameraMaterial; public override bool permissionGranted => true; public override XRSupportedCameraBackgroundRenderingMode requestedBackgroundRenderingMode { get => m_RequestedBackgroundRenderingMode; set => m_RequestedBackgroundRenderingMode = value; } public override XRCameraBackgroundRenderingMode currentBackgroundRenderingMode { get { switch (requestedBackgroundRenderingMode) { case XRSupportedCameraBackgroundRenderingMode.AfterOpaques: return XRCameraBackgroundRenderingMode.AfterOpaques; case XRSupportedCameraBackgroundRenderingMode.BeforeOpaques: case XRSupportedCameraBackgroundRenderingMode.Any: return XRCameraBackgroundRenderingMode.BeforeOpaques; default: return XRCameraBackgroundRenderingMode.None; } } } public override XRSupportedCameraBackgroundRenderingMode supportedBackgroundRenderingMode => XRSupportedCameraBackgroundRenderingMode.Any; public SimulationProvider() { var backgroundShader = Shader.Find(k_BackgroundShaderName); if (backgroundShader == null) Debug.LogError("Cannot create camera background material compatible with the render pipeline"); else m_CameraMaterial = CreateCameraMaterial(k_BackgroundShaderName); } public override void Start() { #if UNITY_EDITOR SimulationSubsystemAnalytics.SubsystemStarted(k_SubsystemId); #endif var xrOrigin = Object.FindAnyObjectByType(); if (xrOrigin == null) { Debug.LogError("An XR Origin is required in the scene, none found."); return; } var xrCamera = xrOrigin.Camera; if (xrCamera == null) { Debug.LogError("No camera found under XROrigin."); return; } var simCameraPoseProvider = SimulationCameraPoseProvider.GetOrCreateSimulationCameraPoseProvider(); m_CameraTextureProvider = CameraTextureProvider.AddTextureProviderToCamera(simCameraPoseProvider.GetComponent(), xrCamera); m_CameraTextureProvider.onTextureReadbackFulfilled += SimulationXRCpuImageApi.OnCameraDataReceived; m_CameraTextureProvider.cameraFrameReceived += CameraFrameReceived; if (m_CameraTextureProvider != null && m_CameraTextureProvider.CameraFrameEventArgs != null) m_CameraTextureFrameEventArgs = (CameraTextureFrameEventArgs)m_CameraTextureProvider.CameraFrameEventArgs; m_Camera = m_CameraTextureProvider.GetComponent(); var lightObject = new GameObject("SimulationLight"); lightObject.transform.parent = m_Camera.transform; lightObject.layer = m_Camera.gameObject.layer; m_CameraTorch = lightObject.AddComponent(); SetupCameraTorch(m_CameraTorch); m_XRCameraConfiguration = new XRCameraConfiguration(IntPtr.Zero, new Vector2Int(m_Camera.pixelWidth, m_Camera.pixelHeight)); m_XRCameraIntrinsics = new XRCameraIntrinsics(); BaseSimulationSceneManager.environmentSetupFinished += OnEnvironmentSetupFinished; m_ExifData = m_Camera.GetComponent(); } public override void Stop() { if (m_CameraTextureProvider != null) { m_CameraTextureProvider.cameraFrameReceived -= CameraFrameReceived; m_CameraTextureProvider.onTextureReadbackFulfilled -= SimulationXRCpuImageApi.OnCameraDataReceived; } BaseSimulationSceneManager.environmentSetupFinished -= OnEnvironmentSetupFinished; } public override void Destroy() { if (m_CameraTextureProvider != null) { Object.Destroy(m_CameraTextureProvider.gameObject); m_CameraTextureProvider = null; } } void OnEnvironmentSetupFinished() { m_MainLight = null; var simulationEnvironmentLights = SimulationSessionSubsystem.simulationSceneManager.simulationEnvironmentLights; foreach (var light in simulationEnvironmentLights) { if (light.simulatedLight.type == LightType.Directional) { if (m_MainLight == null) { m_MainLight = light; } else if (m_MainLight.simulatedLight.intensity < light.simulatedLight.intensity) { Debug.LogWarning("Multiple directional lights were detected in the XR Simulation environment. The light with the highest intensity will be used as the main light."); m_MainLight = light; } } } } public override NativeArray GetConfigurations(XRCameraConfiguration defaultCameraConfiguration, Allocator allocator) { var configs = new NativeArray(1, allocator); configs[0] = m_XRCameraConfiguration; return configs; } public override NativeArray GetTextureDescriptors(XRTextureDescriptor defaultDescriptor, Allocator allocator) { if (m_CameraTextureProvider != null) { m_CameraTextureProvider.TryGetTextureDescriptors(out var descriptors, allocator); return descriptors; } return base.GetTextureDescriptors(defaultDescriptor, allocator); } public override bool TryAcquireLatestCpuImage(out XRCpuImage.Cinfo cameraImageCinfo) { return SimulationXRCpuImageApi.TryAcquireLatestImage(SimulationXRCpuImageApi.ImageType.Camera, out cameraImageCinfo); } void CameraFrameReceived(CameraTextureFrameEventArgs args) { m_CameraTextureFrameEventArgs = args; } public override bool TryGetFrame(XRCameraParams cameraParams, out XRCameraFrame cameraFrame) { if (m_CameraTextureProvider == null) { cameraFrame = new XRCameraFrame(); return false; } XRCameraFrameProperties properties = 0; long timeStamp = default; float averageBrightness = default; float averageColorTemperature = default; Color colorCorrection = default; Matrix4x4 projectionMatrix = default; Matrix4x4 displayMatrix = default; TrackingState trackingState = default; IntPtr nativePtr = default; float averageIntensityInLumens = default; double exposureDuration = default; float exposureOffset = default; float mainLightIntensityInLumens = default; Color mainLightColor = default; Vector3 mainLightDirection = default; SphericalHarmonicsL2 ambientSphericalHarmonics = default; XRTextureDescriptor cameraGrain = default; float noiseIntensity = default; if (m_CameraTextureFrameEventArgs.timestampNs.HasValue) { timeStamp = (long)m_CameraTextureFrameEventArgs.timestampNs; properties |= XRCameraFrameProperties.Timestamp; } if (m_CameraTextureFrameEventArgs.projectionMatrix.HasValue) { projectionMatrix = (Matrix4x4)m_CameraTextureFrameEventArgs.projectionMatrix; properties |= XRCameraFrameProperties.ProjectionMatrix; } if (m_CameraTextureProvider == null || !m_CameraTextureProvider.TryGetLatestImagePtr(out nativePtr)) { cameraFrame = default; m_LastFrameTimestamp = timeStamp; return false; } var simulationEnvironmentLights = SimulationSessionSubsystem.simulationSceneManager?.simulationEnvironmentLights; if (simulationEnvironmentLights?.Count > 0) { averageBrightness = CalculateAverageBrightness(); properties |= XRCameraFrameProperties.AverageBrightness; if (GraphicsSettings.lightsUseLinearIntensity && CalculateAverageColorTemperature(out averageColorTemperature)) { properties |= XRCameraFrameProperties.AverageColorTemperature; } averageIntensityInLumens = CalculateAverageIntensityInLumens(); properties |= XRCameraFrameProperties.AverageIntensityInLumens; if (m_MainLight != null) { colorCorrection = m_MainLight.simulatedLight.color; properties |= XRCameraFrameProperties.ColorCorrection; mainLightIntensityInLumens = UnitConversionUtility.ConvertBrightnessToLumens(m_MainLight.simulatedLight.intensity); properties |= XRCameraFrameProperties.MainLightIntensityLumens; mainLightColor = m_MainLight.simulatedLight.color; properties |= XRCameraFrameProperties.MainLightColor; mainLightDirection = m_MainLight.transform.forward; properties |= XRCameraFrameProperties.MainLightDirection; } } XRCameraFrameExifData exifData = default; if (m_ExifData != null) { XRCameraFrameExifDataProperties exifDataProperties = XRCameraFrameExifDataProperties.None; ColorSpace colorSpace = QualitySettings.activeColorSpace; var camColorSpace = (colorSpace == ColorSpace.Linear) ? XRCameraFrameExifDataColorSpace.Uncalibrated : XRCameraFrameExifDataColorSpace.sRGB; exifDataProperties |= XRCameraFrameExifDataProperties.ColorSpace; var flashStatus = (short)((m_CameraTorch != null && m_CameraTorch.enabled) ? 1 : 0); exifDataProperties |= XRCameraFrameExifDataProperties.Flash; var fNumber = (float)(m_ExifData.apertureValue * m_ExifData.apertureValue) / 4; exifDataProperties |= XRCameraFrameExifDataProperties.FNumber; var exposureTime = Time.deltaTime; // using framerate for exposure time exifDataProperties |= XRCameraFrameExifDataProperties.ExposureTime; var shutterValue = -Mathf.Log(exposureTime) / Mathf.Log(2); exifDataProperties |= XRCameraFrameExifDataProperties.ShutterSpeedValue; // inherited from the above averageBrightness exifDataProperties |= XRCameraFrameExifDataProperties.ApertureValue; // inherited from Simulated Exif Data exifDataProperties |= XRCameraFrameExifDataProperties.ExposureBiasValue; exifDataProperties |= XRCameraFrameExifDataProperties.FocalLength; exifDataProperties |= XRCameraFrameExifDataProperties.PhotographicSensitivity; exifDataProperties |= XRCameraFrameExifDataProperties.MeteringMode; exifData = new XRCameraFrameExifData( IntPtr.Zero, m_ExifData.apertureValue, averageBrightness, exposureTime, shutterValue, m_ExifData.exposureBiasValue, fNumber, m_ExifData.focalLength, flashStatus, camColorSpace, m_ExifData.photographicSensitivity, m_ExifData.meteringMode, exifDataProperties ); properties |= XRCameraFrameProperties.ExifData; } m_LastFrameTimestamp = timeStamp; cameraFrame = new XRCameraFrame( timeStamp, averageBrightness, averageColorTemperature, colorCorrection, projectionMatrix, displayMatrix, trackingState, nativePtr, properties, averageIntensityInLumens, exposureDuration, exposureOffset, mainLightIntensityInLumens, mainLightColor, mainLightDirection, ambientSphericalHarmonics, cameraGrain, noiseIntensity, exifData); return true; } public override bool DoesCurrentCameraSupportTorch() { return m_CameraTorch != null; } public override XRCameraTorchMode currentCameraTorchMode { get => m_CurrentTorchMode; } public override XRCameraTorchMode requestedCameraTorchMode { get => m_RequestedTorchMode; set { m_RequestedTorchMode = value; if (m_CurrentTorchMode == m_RequestedTorchMode) return; SetCurrentTorchModeAfterDelay(value); } } void SetupCameraTorch(Light torch) { torch.enabled = false; torch.type = LightType.Spot; torch.spotAngle = 62; torch.innerSpotAngle = 50; torch.range = 25; torch.transform.localRotation = Quaternion.identity; torch.transform.localPosition = Vector3.zero; torch.transform.localScale = Vector3.one; } async void SetCurrentTorchModeAfterDelay(XRCameraTorchMode value) { await Awaitable.NextFrameAsync(); m_CurrentTorchMode = value; m_CameraTorch.enabled = (m_CurrentTorchMode == XRCameraTorchMode.On); } static float CalculateAverageBrightness() { if (SimulationSessionSubsystem.simulationSceneManager == null) return 0.0f; float sum = 0.0f; var simulationEnvironmentLights = SimulationSessionSubsystem.simulationSceneManager.simulationEnvironmentLights; foreach (var light in simulationEnvironmentLights) { sum += light.simulatedLight.intensity; } return sum / Math.Max(simulationEnvironmentLights.Count, 1); } static bool CalculateAverageColorTemperature(out float result) { if (SimulationSessionSubsystem.simulationSceneManager == null) { result = 0.0f; return false; } float sum = 0.0f; bool usesColorTemperature = false; var simulationEnvironmentLights = SimulationSessionSubsystem.simulationSceneManager.simulationEnvironmentLights; foreach (var light in simulationEnvironmentLights) { if (light.simulatedLight.useColorTemperature) { sum += light.simulatedLight.colorTemperature; usesColorTemperature = true; } } result = sum / Math.Max(simulationEnvironmentLights.Count, 1); return usesColorTemperature; } static float CalculateAverageIntensityInLumens() { if (SimulationSessionSubsystem.simulationSceneManager == null) return 0.0f; float sum = 0.0f; var simulationEnvironmentLights = SimulationSessionSubsystem.simulationSceneManager.simulationEnvironmentLights; foreach (var light in simulationEnvironmentLights) { sum += UnitConversionUtility.ConvertBrightnessToLumens(light.simulatedLight.intensity); } return sum / Math.Max(simulationEnvironmentLights.Count, 1); } public override bool TryGetIntrinsics(out XRCameraIntrinsics cameraIntrinsics) { cameraIntrinsics = m_XRCameraIntrinsics; return true; } } [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.SubsystemRegistration)] static void Register() { var cInfo = new XRCameraSubsystemDescriptor.Cinfo { id = k_SubsystemId, providerType = typeof(SimulationProvider), subsystemTypeOverride = typeof(SimulationCameraSubsystem), supportsCameraConfigurations = true, supportsCameraImage = true, supportsAverageColorTemperature = true, supportsColorCorrection = true, supportsAverageBrightness = true, supportsAverageIntensityInLumens = true, supportsExifData = true, supportsCameraTorchMode = true, }; XRCameraSubsystemDescriptor.Register(cInfo); } } }