Support transparent shadow (#2080)

* feat: support transparent shadow
---------

Co-authored-by: ChenMo <gl3336563@163.com>

e2e/case/shadow-transparent.ts

@@ -0,0 +1,61 @@
+/**
+ * @title Shadow Transparent
+ * @category Shadow
+ */
+
+import {
+  Camera,
+  DirectLight,
+  GLTFResource,
+  Logger,
+  MeshRenderer,
+  PBRMaterial,
+  PrimitiveMesh,
+  ShadowResolution,
+  ShadowType,
+  Vector3,
+  WebGLEngine,
+  WebGLMode
+} from "@galacean/engine";
+import { initScreenshot, updateForE2E } from "./.mockForE2E";
+Logger.enable();
+
+WebGLEngine.create({
+  canvas: "canvas",
+  graphicDeviceOptions: {
+    webGLMode: WebGLMode.WebGL2
+  }
+}).then((engine) => {
+  engine.canvas.resizeByClientSize();
+  const scene = engine.sceneManager.activeScene;
+  const rootEntity = scene.createRootEntity();
+  scene.shadowResolution = ShadowResolution.Medium;
+  scene.shadowDistance = 10;
+  const cameraEntity = rootEntity.createChild("camera_node");
+  cameraEntity.transform.setPosition(0, 2, 3);
+  cameraEntity.transform.lookAt(new Vector3(0));
+  const camera = cameraEntity.addComponent(Camera);
+  const lightEntity = rootEntity.createChild("light_node");
+  const light = lightEntity.addComponent(DirectLight);
+  lightEntity.transform.setPosition(-6, 10, 0);
+  lightEntity.transform.lookAt(new Vector3(0, 0, -10));
+  light.shadowType = ShadowType.Hard;
+
+  const planeEntity = rootEntity.createChild("plane_node");
+  const renderer = planeEntity.addComponent(MeshRenderer);
+  renderer.mesh = PrimitiveMesh.createPlane(engine, 10, 10);
+  const planeMaterial = new PBRMaterial(engine);
+  renderer.setMaterial(planeMaterial);
+
+  engine.resourceManager
+    .load<GLTFResource>("https://mdn.alipayobjects.com/oasis_be/afts/file/A*kgYIRo36270AAAAAAAAAAAAADkp5AQ/bottle.glb")
+    .then((asset) => {
+      const defaultSceneRoot = asset.instantiateSceneRoot();
+      rootEntity.addChild(defaultSceneRoot);
+      defaultSceneRoot.transform.scale.set(0.05, 0.05, 0.05);
+      scene.enableTransparentShadow = true;
+
+      updateForE2E(engine, 500);
+      initScreenshot(engine, camera);
+    });
+});

e2e/config.ts

@@ -111,6 +111,11 @@ export const E2E_CONFIG = {
       category: "Shadow",
       caseFileName: "shadow-basic",
       threshold: 0.2
+    },
+    transparent: {
+      category: "Shadow",
+      caseFileName: "shadow-transparent",
+      threshold: 0.2
     }
   },
   Primitive: {

e2e/fixtures/originImage/Shadow_shadow-transparent.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f58e73beb44e87d8d266b0c0784c3a3e0adeff4d1c19ce10a4b517d31fcca0e7
+size 86887

packages/core/src/RenderPipeline/BasicRenderPipeline.ts

@@ -237,7 +237,17 @@ export class BasicRenderPipeline {
 
     let renderQueueAddedFlags = RenderQueueAddedFlag.None;
     for (let i = 0, n = shaderPasses.length; i < n; i++) {
-      const renderQueueType = (shaderPasses[i]._renderState ?? renderStates[i]).renderQueueType;
+      // Get render queue type
+      let renderQueueType: RenderQueueType;
+      const shaderPass = shaderPasses[i];
+      const renderState = shaderPass._renderState;
+      if (renderState) {
+        renderState._applyRenderQueueByShaderData(shaderPass._renderStateDataMap, element.material.shaderData);
+        renderQueueType = renderState.renderQueueType;
+      } else {
+        renderQueueType = renderStates[i].renderQueueType;
+      }
+
       if (renderQueueAddedFlags & (<RenderQueueAddedFlag>(1 << renderQueueType))) {
         continue;
       }
@@ -280,7 +290,12 @@ export class BasicRenderPipeline {
     program.uploadAll(program.materialUniformBlock, material.shaderData);
     program.uploadUnGroupTextures();
 
-    (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+    (pass._renderState || material.renderState)._applyStates(
+      engine,
+      false,
+      pass._renderStateDataMap,
+      material.shaderData
+    );
     rhi.drawPrimitive(mesh._primitive, mesh.subMesh, program);
   }
 

packages/core/src/RenderPipeline/PipelineUtils.ts

@@ -147,7 +147,7 @@ export class PipelineUtils {
     program.uploadAll(program.materialUniformBlock, blitMaterial.shaderData);
     program.uploadUnGroupTextures();
 
-    (pass._renderState || blitMaterial.renderState)._apply(
+    (pass._renderState || blitMaterial.renderState)._applyStates(
       engine,
       false,
       pass._renderStateDataMap,

packages/core/src/RenderPipeline/RenderQueue.ts

@@ -179,7 +179,7 @@ export class RenderQueue {
           }
 
           const renderState = shaderPass._renderState ?? renderStates[j];
-          renderState._apply(
+          renderState._applyStates(
             engine,
             renderer.entity.transform._isFrontFaceInvert(),
             shaderPass._renderStateDataMap,

packages/core/src/RenderPipeline/SpriteBatcher.ts

@@ -125,7 +125,12 @@ export class SpriteBatcher extends Basic2DBatcher {
       program.uploadAll(program.rendererUniformBlock, renderer.shaderData);
       program.uploadAll(program.materialUniformBlock, material.shaderData);
 
-      (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+      (pass._renderState || material.renderState)._applyStates(
+        engine,
+        false,
+        pass._renderStateDataMap,
+        material.shaderData
+      );
       engine._hardwareRenderer.drawPrimitive(primitive, subMesh, program);
 
       maskManager.postRender(renderer);

packages/core/src/RenderPipeline/SpriteMaskBatcher.ts

@@ -97,7 +97,12 @@ export class SpriteMaskBatcher extends Basic2DBatcher {
       program.uploadAll(program.rendererUniformBlock, renderer.shaderData);
       program.uploadAll(program.materialUniformBlock, material.shaderData);
 
-      (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+      (pass._renderState || material.renderState)._applyStates(
+        engine,
+        false,
+        pass._renderStateDataMap,
+        material.shaderData
+      );
 
       engine._hardwareRenderer.drawPrimitive(primitive, subMesh, program);
     }

packages/core/src/Scene.ts

@@ -73,6 +73,7 @@ export class Scene extends EngineObject {
   private _fogParams: Vector4 = new Vector4();
   private _isActive: boolean = true;
   private _sun: DirectLight | null;
+  private _enableTransparentShadow = false;
 
   /**
    * Whether the scene is active.
@@ -241,6 +242,24 @@ export class Scene extends EngineObject {
     this._sun = light;
   }
 
+  /**
+   * Whether to enable transparent shadow.
+   */
+  get enableTransparentShadow(): boolean {
+    return this._enableTransparentShadow;
+  }
+
+  set enableTransparentShadow(value: boolean) {
+    if (value !== this._enableTransparentShadow) {
+      this._enableTransparentShadow = value;
+      if (value) {
+        this.shaderData.enableMacro("SCENE_ENABLE_TRANSPARENT_SHADOW");
+      } else {
+        this.shaderData.disableMacro("SCENE_ENABLE_TRANSPARENT_SHADOW");
+      }
+    }
+  }
+
   /**
    * Create scene.
    * @param engine - Engine

packages/core/src/material/BaseMaterial.ts

@@ -8,6 +8,9 @@ import { RenderFace } from "./enums/RenderFace";
 import { Material } from "./Material";
 
 export class BaseMaterial extends Material {
+  /** @internal */
+  static _shadowCasterRenderQueueProp = ShaderProperty.getByName("material_ShadowCasterRenderQueue");
+
   protected static _baseTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_BASETEXTURE");
   protected static _normalTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_NORMALTEXTURE");
   protected static _emissiveTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_EMISSIVETEXTURE");
@@ -63,7 +66,7 @@ export class BaseMaterial extends Material {
   }
 
   /**
-   * Whethor transparent of first shader pass render state.
+   * Whether transparent of first shader pass render state.
    */
   get isTransparent(): boolean {
     return this._isTransparent;
@@ -72,6 +75,20 @@ export class BaseMaterial extends Material {
   set isTransparent(value: boolean) {
     if (value !== this._isTransparent) {
       this.setIsTransparent(0, value);
+
+      const { shaderData } = this;
+      if (value) {
+        // Use alpha test queue to simulate transparent shadow
+        shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+      } else {
+        const alphaCutoff = shaderData.getFloat(BaseMaterial._alphaCutoffProp);
+        if (alphaCutoff) {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+        } else {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
+        }
+      }
+
       this._isTransparent = value;
     }
   }
@@ -106,8 +123,14 @@ export class BaseMaterial extends Material {
     if (shaderData.getFloat(BaseMaterial._alphaCutoffProp) !== value) {
       if (value) {
         shaderData.enableMacro(BaseMaterial._alphaCutoffMacro);
+        shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
       } else {
         shaderData.disableMacro(BaseMaterial._alphaCutoffMacro);
+        if (this._isTransparent) {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+        } else {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
+        }
       }
 
       const { renderStates } = this;
@@ -123,7 +146,6 @@ export class BaseMaterial extends Material {
             : RenderQueueType.Opaque;
         }
       }
-
       shaderData.setFloat(BaseMaterial._alphaCutoffProp, value);
     }
   }
@@ -149,7 +171,10 @@ export class BaseMaterial extends Material {
    */
   constructor(engine: Engine, shader: Shader) {
     super(engine, shader);
-    this.shaderData.setFloat(BaseMaterial._alphaCutoffProp, 0);
+
+    const { shaderData } = this;
+    shaderData.setFloat(BaseMaterial._alphaCutoffProp, 0);
+    shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
   }
 
   /**
@@ -163,19 +188,21 @@ export class BaseMaterial extends Material {
       throw "Pass should less than pass count.";
     }
     const renderState = renderStates[passIndex];
+    const { shaderData } = this;
 
     if (isTransparent) {
       renderState.blendState.targetBlendState.enabled = true;
       renderState.depthState.writeEnabled = false;
       renderState.renderQueueType = RenderQueueType.Transparent;
-      this.shaderData.enableMacro(BaseMaterial._transparentMacro);
+      shaderData.enableMacro(BaseMaterial._transparentMacro);
     } else {
       renderState.blendState.targetBlendState.enabled = false;
       renderState.depthState.writeEnabled = true;
-      renderState.renderQueueType = this.shaderData.getFloat(BaseMaterial._alphaCutoffProp)
+
+      renderState.renderQueueType = shaderData.getFloat(BaseMaterial._alphaCutoffProp)
         ? RenderQueueType.AlphaTest
         : RenderQueueType.Opaque;
-      this.shaderData.disableMacro(BaseMaterial._transparentMacro);
+      shaderData.disableMacro(BaseMaterial._transparentMacro);
     }
   }
 

packages/core/src/shader/ShaderPool.ts

@@ -1,4 +1,5 @@
 import { PipelineStage } from "../RenderPipeline/enums/PipelineStage";
+import { BaseMaterial } from "../material/BaseMaterial";
 import blitFs from "../shaderlib/extra/Blit.fs.glsl";
 import blitVs from "../shaderlib/extra/Blit.vs.glsl";
 import skyProceduralFs from "../shaderlib/extra/SkyProcedural.fs.glsl";
@@ -26,6 +27,8 @@ import unlitFs from "../shaderlib/extra/unlit.fs.glsl";
 import unlitVs from "../shaderlib/extra/unlit.vs.glsl";
 import { Shader } from "./Shader";
 import { ShaderPass } from "./ShaderPass";
+import { RenderStateElementKey } from "./enums/RenderStateElementKey";
+import { RenderState } from "./state";
 
 /**
  * Internal shader pool.
@@ -36,6 +39,10 @@ export class ShaderPool {
     const shadowCasterPass = new ShaderPass("ShadowCaster", shadowMapVs, shadowMapFs, {
       pipelineStage: PipelineStage.ShadowCaster
     });
+    shadowCasterPass._renderState = new RenderState();
+    shadowCasterPass._renderStateDataMap[RenderStateElementKey.RenderQueueType] =
+      BaseMaterial._shadowCasterRenderQueueProp;
+
     const depthOnlyPass = new ShaderPass("DepthOnly", depthOnlyVs, depthOnlyFs, {
       pipelineStage: PipelineStage.DepthOnly
     });

packages/core/src/shader/state/RenderState.ts

@@ -30,30 +30,15 @@ export class RenderState {
 
   /**
    * @internal
-   * @todo Should merge when we can delete material render state.
    */
-  _applyShaderDataValue(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
-    this.blendState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.depthState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.stencilState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.rasterState._applyShaderDataValue(renderStateDataMap, shaderData);
-
-    const renderQueueType = renderStateDataMap[RenderStateElementKey.RenderQueueType];
-    if (renderQueueType !== undefined) {
-      this.renderQueueType = shaderData.getFloat(renderQueueType) ?? RenderQueueType.Opaque;
-    }
-  }
-
-  /**
-   * @internal
-   */
-  _apply(
+  _applyStates(
     engine: Engine,
     frontFaceInvert: boolean,
     renderStateDataMap: Record<number, ShaderProperty>,
     shaderData: ShaderData
   ): void {
-    renderStateDataMap && this._applyShaderDataValue(renderStateDataMap, shaderData);
+    // @todo: Should merge when we can delete material render state
+    renderStateDataMap && this._applyStatesByShaderData(renderStateDataMap, shaderData);
     const hardwareRenderer = engine._hardwareRenderer;
     const lastRenderState = engine._lastRenderState;
     const context = engine._renderContext;
@@ -66,4 +51,22 @@ export class RenderState {
       context.flipProjection ? !frontFaceInvert : frontFaceInvert
     );
   }
+
+  /**
+   * @internal
+   * @todo Should merge when we can delete material render state
+   */
+  _applyRenderQueueByShaderData(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
+    const renderQueueType = renderStateDataMap[RenderStateElementKey.RenderQueueType];
+    if (renderQueueType !== undefined) {
+      this.renderQueueType = shaderData.getFloat(renderQueueType) ?? RenderQueueType.Opaque;
+    }
+  }
+
+  private _applyStatesByShaderData(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
+    this.blendState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.depthState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.stencilState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.rasterState._applyShaderDataValue(renderStateDataMap, shaderData);
+  }
 }

packages/core/src/shaderlib/extra/shadow-map.fs.glsl

@@ -1,25 +1,52 @@
 #ifdef ENGINE_NO_DEPTH_TEXTURE
-/**
- * Decompose and save depth value.
-*/
-vec4 pack (float depth) {
-  // Use rgba 4 bytes with a total of 32 bits to store the z value, and the accuracy of 1 byte is 1/256.
-  const vec4 bitShift = vec4(1.0, 256.0, 256.0 * 256.0, 256.0 * 256.0 * 256.0);
-  const vec4 bitMask = vec4(1.0/256.0, 1.0/256.0, 1.0/256.0, 0.0);
+  /**
+   * Decompose and save depth value.
+  */
+  vec4 pack (float depth) {
+    // Use rgba 4 bytes with a total of 32 bits to store the z value, and the accuracy of 1 byte is 1/256.
+    const vec4 bitShift = vec4(1.0, 256.0, 256.0 * 256.0, 256.0 * 256.0 * 256.0);
+    const vec4 bitMask = vec4(1.0/256.0, 1.0/256.0, 1.0/256.0, 0.0);
 
-  vec4 rgbaDepth = fract(depth * bitShift); // Calculate the z value of each point
+    vec4 rgbaDepth = fract(depth * bitShift); // Calculate the z value of each point
 
-  // Cut off the value which do not fit in 8 bits
-  rgbaDepth -= rgbaDepth.gbaa * bitMask;
+    // Cut off the value which do not fit in 8 bits
+    rgbaDepth -= rgbaDepth.gbaa * bitMask;
 
-  return rgbaDepth;
-}
+    return rgbaDepth;
+  }
 #endif
 
+
+uniform vec4 material_BaseColor;
+uniform sampler2D material_BaseTexture;
+uniform float material_AlphaCutoff;
+varying vec2 v_uv;
+
 void main() {
-#ifdef ENGINE_NO_DEPTH_TEXTURE
-    gl_FragColor = pack(gl_FragCoord.z);
-#else
-    gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);
-#endif
+  #if defined(MATERIAL_IS_ALPHA_CUTOFF) || (defined(SCENE_ENABLE_TRANSPARENT_SHADOW) && defined(MATERIAL_IS_TRANSPARENT))
+    float alpha = material_BaseColor.a;
+    #ifdef MATERIAL_HAS_BASETEXTURE
+        alpha *= texture2D(material_BaseTexture, v_uv).a;
+    #endif
+  
+    #ifdef MATERIAL_IS_ALPHA_CUTOFF
+      if(alpha < material_AlphaCutoff){
+        discard;
+      }
+    #endif
+  
+    #if defined(SCENE_ENABLE_TRANSPARENT_SHADOW) && defined(MATERIAL_IS_TRANSPARENT)
+      // Interleaved gradient noise
+      float noise = fract(52.982919 * fract(dot(vec2(0.06711, 0.00584), gl_FragCoord.xy)));
+      if (alpha <= noise) {
+        discard;
+      };
+    #endif
+  #endif
+
+  #ifdef ENGINE_NO_DEPTH_TEXTURE
+      gl_FragColor = pack(gl_FragCoord.z);
+  #else
+      gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);
+  #endif
 }

packages/core/src/shaderlib/extra/shadow-map.vs.glsl

@@ -2,6 +2,7 @@
 #include <common_vert>
 #include <blendShape_input>
 #include <normal_share>
+#include <uv_share>
 uniform mat4 camera_VPMat;
 uniform vec2 scene_ShadowBias; // x: depth bias, y: normal bias
 uniform vec3 scene_LightDirection;
@@ -24,6 +25,7 @@ void main() {
     #include <begin_normal_vert>
     #include <blendShape_vert>
     #include <skinning_vert>
+    #include <uv_vert>
     
     vec4 positionWS = renderer_ModelMat * position;
 

packages/core/src/shaderlib/pbr/brdf.glsl

@@ -58,8 +58,8 @@ float D_GGX(float alpha, float dotNH ) {
     // https://blog.selfshadow.com/publications/s2012-shading-course/burley/s2012_pbs_disney_brdf_notes_v3.pdf Addenda
     float D_GGX_Anisotropic(float at, float ab, float ToH, float BoH, float NoH) {
         float a2 = at * ab;
-        highp vec3 d = vec3(ab * ToH, at * BoH, a2 * NoH);
-        highp float d2 = dot(d, d);
+        vec3 d = vec3(ab * ToH, at * BoH, a2 * NoH);
+        float d2 = dot(d, d);
         float b2 = a2 / d2;
         return a2 * b2 * b2 * RECIPROCAL_PI;
     }

packages/core/src/shadow/CascadedShadowCasterPass.ts

@@ -89,7 +89,7 @@ export class CascadedShadowCasterPass extends PipelinePass {
       _shadowMatrices: shadowMatrices
     } = this;
 
-    const { opaqueQueue, alphaTestQueue, transparentQueue } = camera._renderPipeline._cullingResults;
+    const { opaqueQueue, alphaTestQueue } = camera._renderPipeline._cullingResults;
 
     const scene = camera.scene;
     const componentsManager = scene._componentsManager;
@@ -210,7 +210,6 @@ export class CascadedShadowCasterPass extends PipelinePass {
       splitBoundSpheres[offset + 3] = radius * radius;
       opaqueQueue.clear();
       alphaTestQueue.clear();
-      transparentQueue.clear();
       const renderers = componentsManager._renderers;
       const elements = renderers._elements;
       for (let k = renderers.length - 1; k >= 0; --k) {

packages/core/src/sky/Sky.ts

@@ -122,7 +122,7 @@ export class Sky {
     program.uploadAll(program.materialUniformBlock, materialShaderData);
     program.uploadUnGroupTextures();
 
-    renderState._apply(engine, false, pass._renderStateDataMap, materialShaderData);
+    renderState._applyStates(engine, false, pass._renderStateDataMap, materialShaderData);
     rhi.drawPrimitive(mesh._primitive, mesh.subMesh, program);
     cameraShaderData.setMatrix(RenderContext.vpMatrixProperty, originViewProjMatrix);
   }