ARDK UPM 3.11

Author: sean-niantic

Assets/Shaders/LightshipScanningStripes.shader

@@ -8,6 +8,7 @@ Shader "Unlit/LightshipScanningStripes"
         _PositionAndConfidenceTex("RaycastPositionAndConfidence", 2D) = "white" {}
         _ScreenOrientation("ScreenOrientation", int) = 0
         _StripeColor ("StripeColor", Color) = (1,0,0,1)
+        _ArCameraTex ("ArCameraImage", 2D) = "white" {}
     }
     SubShader
     {
@@ -24,8 +25,10 @@ Shader "Unlit/LightshipScanningStripes"
             fixed4 _StripeColor;
             sampler2D _MainTex;
             sampler2D _ColorTex;
+            sampler2D _ArCameraTex;
             float4 _MainTex_TexelSize;
             float4 _ColorTex_TexelSize;
+            float4 _ArCameraTex_TexelSize;
             int _ScreenOrientation;
 
             struct v2f
@@ -66,17 +69,33 @@ Shader "Unlit/LightshipScanningStripes"
                 return uv * scale + 0.5f;
             }
 
+            // Returns cropped camera buffer coordinates to match the raycast buffer.
+            inline float2 CropCameraImage(float2 uv)
+            {
+                uv -= 0.5f;
+                float outAspect = _ColorTex_TexelSize.z / _ColorTex_TexelSize.w;
+                // Inverted aspect to account for raycast rotated to landscape left
+                float inAspect = _ArCameraTex_TexelSize.w / _ArCameraTex_TexelSize.z;
+
+                // If the camera image is wider than the raycast, crop the width.
+                float uScaled = inAspect > outAspect ? uv.x * outAspect / inAspect : uv.x;
+                // If the camera image is taller than the raycast, crop the height.
+                float vScaled = inAspect < outAspect ? uv.y * inAspect / outAspect : uv.y;
+                return float2(uScaled, vScaled) + 0.5f;
+            }
+
             fixed4 frag (v2f i, UNITY_VPOS_TYPE screenPos : VPOS) : SV_Target
             {
                 // Sample the raycast color from _ColorTex.
                 fixed4 color = tex2D(_ColorTex, GetRaycastSampleUV(i.uv));
+                fixed4 cameraColor = tex2D(_ArCameraTex, CropCameraImage(i.uv));
 
                 // Compute the color of the diagonal stripes.
                 unsigned int stripeCoord = ((int) (screenPos.x + _ScreenParams.y - screenPos.y)) & 15;
                 fixed4 stripeColor = (stripeCoord < 10) ? _StripeColor : 1;
 
-                // Blend raycast color over the stripe color.
-                return color * color.a + stripeColor * (1 - color.a);
+                // Blend raycast color, AR camera view and the stripe color.
+                return cameraColor * color.a / 2 + color * color.a / 2 + stripeColor * (1 - color.a);
             }
             ENDCG
         }

Assets/Shaders/OcclusionMesh.shader

@@ -0,0 +1,167 @@
+Shader "Lightship/OcclusionMesh"
+{
+    Properties
+    {
+        // Retrievable properties
+        _Depth ("DepthTexture", 2D) = "black" {}
+        _ColorMask ("Color Mask", Float) = 0
+    }
+    SubShader
+    {
+        Tags { "RenderType"="Opaque" }
+        LOD 100
+
+        Pass
+        {
+            Offset 1, 1
+            ColorMask [_ColorMask]
+
+            CGPROGRAM
+            #pragma vertex vert
+            #pragma fragment frag
+            #pragma target 3.5
+
+            #pragma multi_compile_local __ FEATURE_SUPPRESSION
+            #pragma multi_compile_local __ FEATURE_STABILIZATION
+
+            #include "UnityCG.cginc"
+
+            struct appdata
+            {
+                float4 vertex : POSITION;
+                uint vid : SV_VertexID;
+
+                UNITY_VERTEX_INPUT_INSTANCE_ID
+            };
+
+            struct v2f
+            {
+                float4 pos : SV_POSITION;
+                half4 color : COLOR;
+
+                UNITY_VERTEX_INPUT_INSTANCE_ID
+                UNITY_VERTEX_OUTPUT_STEREO
+            };
+
+            sampler2D _Depth;
+
+#ifdef FEATURE_STABILIZATION
+            sampler2D _FusedDepth;
+            float4x4 _FusedDepthTransform;
+#endif
+
+#ifdef FEATURE_SUPPRESSION
+            sampler2D _Suppression;
+            float4x4 _SuppressionTransform;
+#endif
+
+            // Matrices
+            float4 _Intrinsics;
+            float4x4 _Extrinsics;
+
+            float _UnityCameraForwardScale;
+            float _StabilizationThreshold;
+            int _ImageWidth;
+            int _ImageHeight;
+
+            inline float ScaleEyeDepth(float d)
+            {
+                d = _UnityCameraForwardScale > 0.0 ? _UnityCameraForwardScale * d : d;
+                return d < _ProjectionParams.y ? 0.0f : d;
+            }
+
+            inline float4 WorldToClipPos(float3 posWorld)
+            {
+              float4 clipPos;
+              #if defined(STEREO_CUBEMAP_RENDER_ON)
+                float3 offset = ODSOffset(posWorld, unity_HalfStereoSeparation.x);
+                clipPos = mul(UNITY_MATRIX_VP, float4(posWorld + offset, 1.0));
+              #else
+                clipPos = mul(UNITY_MATRIX_VP, float4(posWorld, 1.0));
+              #endif
+              return clipPos;
+            }
+
+            v2f vert (appdata v)
+            {
+                v2f o;
+
+                UNITY_SETUP_INSTANCE_ID(v);
+                UNITY_INITIALIZE_OUTPUT(v2f, o);
+                UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
+
+                // Get the pixel coordinates for the vertex
+                uint2 pixel = uint2(v.vid % _ImageWidth, v.vid / _ImageHeight);
+
+                // Calculate UV coordinates for the vertex
+                float4 uv = float4(pixel.x / (float)_ImageWidth, pixel.y / (float)_ImageHeight, 0.0f, 0.0f);
+                uv.y = 1.0f - uv.y; // Flip y-axis (native image is mirrored)
+
+                // Sample depth
+                float eye_depth = tex2Dlod(_Depth, uv).r;
+
+#ifdef FEATURE_STABILIZATION
+                // Calculate UV coordinates for the fused depth
+                // This is not a native image, so we have to un-invert the y-axis
+                float4 fusedDepthUV = mul(_FusedDepthTransform, float4(uv.x, 1.0f - uv.y, 1.0f, 1.0f));
+
+                // Sample fused depth
+                const float eps = 0.1f;
+                float fused_depth_linear_eye =
+                  tex2Dlod(_FusedDepth, float4(fusedDepthUV.x / fusedDepthUV.z, fusedDepthUV.y / fusedDepthUV.z, 0, 0)) + eps;
+
+                // Determine the depth value
+                bool useFrameDepth = (abs(fused_depth_linear_eye - eye_depth) / fused_depth_linear_eye) >= _StabilizationThreshold;
+                eye_depth = useFrameDepth ? eye_depth : fused_depth_linear_eye;
+#endif
+
+                // Convert from image space to view space
+                float4 view_position;
+                view_position.x = (pixel.x - _Intrinsics.z) * eye_depth / _Intrinsics.x;
+                view_position.y = (pixel.y - _Intrinsics.w) * eye_depth / _Intrinsics.y;
+                view_position.z = ScaleEyeDepth(eye_depth);
+                view_position.w = 1.0f;
+
+#ifdef FEATURE_SUPPRESSION
+                // Sample the suppression mask
+                float4 suppression_uv = mul(_SuppressionTransform, float4(uv.x, 1.0f - uv.y, 1.0f, 1.0f));
+                float mask = tex2Dlod(_Suppression, float4(suppression_uv.x / suppression_uv.z, suppression_uv.y / suppression_uv.z, 0, 0)).r;
+
+                // Determine whether to offset the vertex
+                float scalar = mask > 0.5f ? 1000.0f : 1.0f;
+                view_position = view_position * scalar;
+                view_position.w = 1.0f;
+#endif
+
+                // Transform to world space
+                float4 world_position = mul(_Extrinsics, view_position);
+
+                // Transform to clip space
+                o.pos = WorldToClipPos(world_position.xyz);
+
+                // Set colors to debug UV coordinates
+                o.color.x = uv.x;
+                o.color.z = uv.y;
+
+                // Set colors to debug eye depth
+                const float max_view_disp = 1.0f;
+                o.color.y = (1.0f / eye_depth) / max_view_disp;
+
+                // For debug visualization, we need opaque
+                o.color.w = 1.0f;
+
+                return o;
+            }
+
+            half4 frag(v2f i) : COLOR
+            {
+                UNITY_SETUP_STEREO_EYE_INDEX_POST_VERTEX(i);
+
+                // Use the input color from the vertex, in the event we're using debug visualization.
+                return i.color;
+            }
+
+            ENDCG
+        }
+    }
+}

Assets/Shaders/OcclusionMesh.shader.meta

@@ -1,11 +1,9 @@
-Shader "Lightship/OcclusionExtension"
+Shader "Lightship/ZBufferOcclusion"
 {
     Properties
     {
+        // Retrievable properties
         _Depth ("DepthTexture", 2D) = "black" {}
-        _FusedDepth("FusedDepthTexture", 2D) = "black" {}
-        _Suppression ("SuppressionTexture", 2D) = "black" {}
-        _StabilizationThreshold ("Stabilization Threshold", Range(0.0, 1.0)) = 0.5
     }
     SubShader
     {

Assets/Shaders/OcclusionExtension.shader renamed to Assets/Shaders/ZBufferOcclusion.shader

@@ -8,6 +8,7 @@
 using Niantic.Lightship.AR.Utilities.Logging;
 using Niantic.Lightship.AR.Loader;
 using Niantic.Lightship.AR.Occlusion;
+using Niantic.Lightship.AR.Occlusion.Features;
 using Niantic.Lightship.AR.Subsystems.Playback;
 using Unity.EditorCoroutines.Editor;
 using UnityEditor;
@@ -192,8 +193,9 @@ private void PreprocessBuild(BuildReport report)
                     BuildHelper.AddBackgroundShaderToProject(backgroundShaderName);
                 }
 
-                BuildHelper.AddBackgroundShaderToProject(LightshipOcclusionExtension.DefaultShaderName);
-                BuildHelper.AddBackgroundShaderToProject(LightshipFusedDepthCamera.DefaultShaderName);
+                BuildHelper.AddBackgroundShaderToProject(ZBufferOcclusion.RequiredShaderName);
+                BuildHelper.AddBackgroundShaderToProject(OcclusionMesh.RequiredShaderName);
+                BuildHelper.AddBackgroundShaderToProject(Stabilization.RequiredShaderName);
 
                 // TODO: Things that ARKit and ARCore BuildProcessor implementations do
                 // - Check camera usage description