Skip to content

Viewer fixes for IBL Shadows (Draft) #16659

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed

Viewer fixes for IBL Shadows (Draft) #16659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f2ca061
Fix compile error for WebGPU and animated meshes
MiiBond May 22, 2025
e451294
Fix lint error
MiiBond May 22, 2025
f9880e1
Fixes for IBL shadows debug modes
MiiBond May 23, 2025
0bab3bc
Add way to remove all meshes from IBL shadows pipeline
MiiBond May 23, 2025
b54934d
Fixes for IBL shadows in viewer
MiiBond May 23, 2025
73fcc71
Merge branch 'master' of github.com:BabylonJS/Babylon.js into mbond/f…
MiiBond May 23, 2025
d8472df
Fix snapshot rendering with ibl shadows
ryantrem May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions packages/dev/core/src/Rendering/IBLShadows/iblShadowsRenderPipeline.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,13 @@ export class IblShadowsRenderPipeline extends PostProcessRenderPipeline {
}
}

/**
* Clear the list of shadow-casting meshes. This will remove all meshes from the list
*/
public clearShadowCastingMeshes(): void {
this._shadowCastingMeshes.length = 0;
}

/**
* The exponent of the resolution of the voxel shadow grid. Higher resolutions will result in sharper
* shadows but are more expensive to compute and require more memory.
Expand Down Expand Up @@ -881,6 +888,7 @@ export class IblShadowsRenderPipeline extends PostProcessRenderPipeline {
},
};
this._gbufferDebugPass = new PostProcess("iblShadowGBufferDebug", "iblShadowGBufferDebug", options);
this._gbufferDebugPass.samples = (this.engine as any).currentSampleCount || 1;
this._gbufferDebugPass.autoClear = false;
this._gbufferDebugPass.onApplyObservable.add((effect) => {
const depthIndex = this._geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.SCREENSPACE_DEPTH_TEXTURE_TYPE);
Expand Down Expand Up @@ -1091,6 +1099,19 @@ export class IblShadowsRenderPipeline extends PostProcessRenderPipeline {
}
}

/**
* Clear the list of materials that receive shadows. This will remove all materials from the list
*/
public clearShadowReceivingMaterials() {
for (const mat of this._materialsWithRenderPlugin) {
const plugin = mat.pluginManager?.getPlugin<IBLShadowsPluginMaterial>(IBLShadowsPluginMaterial.Name);
if (plugin) {
plugin.isEnabled = false;
}
}
this._materialsWithRenderPlugin.length = 0;
}

protected _addShadowSupportToMaterial(material: Material) {
if (!(material instanceof PBRBaseMaterial) && !(material instanceof StandardMaterial)) {
return;
Expand Down
2 changes: 1 addition & 1 deletion packages/dev/core/src/Rendering/IBLShadows/iblShadowsSpatialBlurPass.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ export class _IblShadowsSpatialBlurPass {
const debugOptions: PostProcessOptions = {
width: this._engine.getRenderWidth(),
height: this._engine.getRenderHeight(),
textureFormat: Constants.TEXTUREFORMAT_R,
textureFormat: Constants.TEXTUREFORMAT_RGBA,
textureType: Constants.TEXTURETYPE_UNSIGNED_BYTE,
samplingMode: Constants.TEXTURE_NEAREST_SAMPLINGMODE,
uniforms: ["sizeParams"],
Expand Down
7 changes: 4 additions & 3 deletions packages/dev/core/src/ShadersWGSL/ShadersInclude/bonesDeclaration.fx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@
uniform boneTextureWidth : f32;
#else
uniform mBones : array<mat4x4f, BonesPerMesh>;
#ifdef BONES_VELOCITY_ENABLED
uniform mPreviousBones : array<mat4x4f, BonesPerMesh>;
#endif
#endif

#ifdef BONES_VELOCITY_ENABLED
uniform mPreviousBones : array<mat4x4f, BonesPerMesh>;
#endif

#ifdef BONETEXTURE
Expand Down
44 changes: 22 additions & 22 deletions packages/dev/core/src/ShadersWGSL/iblShadowGBufferDebug.fragment.fx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,39 @@
varying vUV: vec2f;
varying vUV : vec2f;

var textureSamplerSampler: sampler;
var textureSampler: texture_2d<f32>;
var depthSampler: sampler;
var depthTexture: texture_2d<f32>;
var normalSampler: sampler;
var normalTexture: texture_2d<f32>;
var positionSampler: sampler;
var positionTexture: texture_2d<f32>;
var velocitySampler: sampler;
var velocityTexture: texture_2d<f32>;
uniform sizeParams: vec4f;
uniform maxDepth: f32;
var textureSamplerSampler : sampler;
var textureSampler : texture_2d<f32>;
var depthSamplerSampler : sampler;
var depthSampler : texture_2d<f32>;
var normalSamplerSampler : sampler;
var normalSampler : texture_2d<f32>;
var positionSamplerSampler : sampler;
var positionSampler : texture_2d<f32>;
var velocitySamplerSampler : sampler;
var velocitySampler : texture_2d<f32>;
uniform sizeParams : vec4f;

#define offsetX uniforms.sizeParams.x
#define offsetY uniforms.sizeParams.y
#define widthScale uniforms.sizeParams.z
#define heightScale uniforms.sizeParams.w

@fragment
fn main(input: FragmentInputs) -> FragmentOutputs {
var uv: vec2f =
vec2f((offsetX + input.vUV.x) * widthScale, (offsetY + input.vUV.y) * heightScale);
@fragment fn main(input : FragmentInputs)->FragmentOutputs {
var uv : vec2f =
vec2f((offsetX + input.vUV.x) * widthScale, (offsetY + input.vUV.y) * heightScale);
var backgroundColour: vec4f = textureSample(textureSampler, textureSamplerSampler, input.vUV).rgba;
var depth: vec4f = textureSample(depthTexture, depthSampler, input.vUV);
var worldNormal: vec4f = textureSample(normalTexture, normalSampler, input.vUV);
var worldPosition: vec4f = textureSample(positionTexture, positionSampler, input.vUV);
var velocityLinear: vec4f = textureSample(velocityTexture, velocitySampler, input.vUV);
var depth : vec4f = textureSample(depthSampler, depthSamplerSampler, input.vUV);
var worldNormal: vec4f = textureSample(normalSampler, normalSamplerSampler, input.vUV);
var worldPosition : vec4f = textureSample(positionSampler, positionSamplerSampler, input.vUV);
var velocityLinear : vec4f = textureSample(velocitySampler, velocitySamplerSampler, input.vUV);
if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) {
fragmentOutputs.color = backgroundColour;
} else {
if (uv.x <= 0.25) {
fragmentOutputs.color = vec4f(depth.rgb, 1.0);
} else if (uv.x <= 0.5) {
velocityLinear = vec4f(velocityLinear.r * 0.5 + 0.5, velocityLinear.g * 0.5 + 0.5, velocityLinear.b, velocityLinear.a);
velocityLinear =
vec4f(velocityLinear.r * 0.5 + 0.5, velocityLinear.g * 0.5 + 0.5,
velocityLinear.b, velocityLinear.a);
fragmentOutputs.color = vec4f(velocityLinear.rgb, 1.0);
} else if (uv.x <= 0.75) {
fragmentOutputs.color = vec4f(worldPosition.rgb, 1.0);
Expand Down
44 changes: 25 additions & 19 deletions packages/tools/viewer/src/viewer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,7 @@ import { registerBuiltInLoaders } from "loaders/dynamic";

export type ResetFlag = "source" | "environment" | "camera" | "animation" | "post-processing" | "material-variant" | "shadow";

// TODO: Include "high" when remaining IBL shadow issues are resolved.
const shadowQualityOptions = ["none", "normal" /*, "high"*/] as const;
const shadowQualityOptions = ["none", "normal", "high"] as const;
export type ShadowQuality = (typeof shadowQualityOptions)[number];

const toneMappingOptions = ["none", "standard", "aces", "neutral"] as const;
Expand Down Expand Up @@ -814,8 +813,7 @@ export class Viewer implements IDisposable {
private _camerasAsHotSpots = false;
private _hotSpots: Record<string, HotSpot> = this._options?.hotSpots ?? {};

// TODO: Remove the `| "high"` once the IBL shadow issues are resolved.
private _shadowQuality: ShadowQuality | "high" = this._options?.shadowConfig?.quality ?? DefaultViewerOptions.shadowConfig.quality;
private _shadowQuality: ShadowQuality = this._options?.shadowConfig?.quality ?? DefaultViewerOptions.shadowConfig.quality;
private readonly _shadowState: ShadowState = {};

public constructor(
Expand Down Expand Up @@ -905,6 +903,7 @@ export class Viewer implements IDisposable {
this._scene.skipPointerUpPicking = true;
this._scene.skipPointerMovePicking = true;
this._snapshotHelper = new SnapshotRenderingHelper(this._scene, { morphTargetsNumMaxInfluences: 30 });
// this._snapshotHelper.showDebugLogs = true;
this._beforeRenderObserver = this._scene.onBeforeRenderObservable.add(() => {
this._snapshotHelper.updateMesh(this._scene.meshes);
});
Expand Down Expand Up @@ -1548,19 +1547,11 @@ export class Viewer implements IDisposable {
await this.loadEnvironment("auto", { lighting: true, skybox: false });
}

if (this._shadowQuality === "normal") {
await this._updateShadowMap(abortController.signal);
} else if (this._shadowQuality === "high") {
const isWebGPU = this._scene.getEngine().isWebGPU;
// there is some issue with meshes with indices, so disable environment shadows for now
const hasAnyAnimationOrIndices = this._loadedModelsBacking.some(
(model) => model.assetContainer.animationGroups.length > 0 && model.assetContainer.meshes.some((mesh) => mesh.getIndices() !== null)
);

if (this._loadedModelsBacking.length > 0 && !(isWebGPU && hasAnyAnimationOrIndices)) {
if (this._loadedModelsBacking.length > 0) {
if (this._shadowQuality === "normal") {
await this._updateShadowMap(abortController.signal);
} else if (this._shadowQuality === "high") {
await this._updateEnvShadow(abortController.signal);
} else {
this._log("Environment shadows are not supported in WebGPU with animated meshes.");
}
}
}
Expand Down Expand Up @@ -1595,12 +1586,18 @@ export class Viewer implements IDisposable {
if (this._shadowState.high) {
if (this._shadowState.high.renderTimer != null) {
clearTimeout(this._shadowState.high.renderTimer);
} else {
// Only disable if a timeout is not pending, otherwise it has already been called without a paired enable call.
this._snapshotHelper.disableSnapshotRendering();
}

this._shadowState.high.shouldRender = true;
const onRenderTimeout = () => {
if (this._shadowState.high) {
this._shadowState.high.shouldRender = false;
this._shadowState.high.renderTimer = null;
}
this._snapshotHelper.enableSnapshotRendering();
};
this._shadowState.high.renderTimer = setTimeout(
onRenderTimeout,
Expand Down Expand Up @@ -1637,24 +1634,26 @@ export class Viewer implements IDisposable {

const updateMaterial = () => {
if (this._shadowState.high) {
this._snapshotHelper.disableSnapshotRendering();
const { pipeline, groundMaterial, ground } = this._shadowState.high;
groundMaterial?.setVector2("renderTargetSize", new Vector2(this._scene.getEngine().getRenderWidth(), this._scene.getEngine().getRenderHeight()));
groundMaterial?.setFloat("shadowOpacity", pipeline.shadowOpacity);
groundMaterial?.setTexture("shadowTexture", pipeline._getAccumulatedTexture());
const groundSize = groundFactor * pipeline?.voxelGridSize;
ground?.scaling.set(groundSize, groundSize, groundSize);
this._snapshotHelper.enableSnapshotRendering();
this._markSceneMutated();
}
};

this._snapshotHelper.disableSnapshotRendering();

let high = this._shadowState.high;
if (!high) {
const pipeline = new IblShadowsRenderPipeline(
"ibl shadows",
this._scene,
{
resolutionExp: 5,
resolutionExp: 6,
sampleDirections: 3,
ssShadowsEnabled: true,
shadowRemanence: 0.7,
Expand Down Expand Up @@ -1693,7 +1692,7 @@ export class Viewer implements IDisposable {

updateMaterial();

pipeline.onShadowTextureReadyObservable.addOnce(updateMaterial);
pipeline.onShadowTextureReadyObservable.add(updateMaterial);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this change needed?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess regardless we don't really need to track and remove the observer, since we would do that at exactly the same time as when we dispose the entire pipeline anyway (which clears observers), correct?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, I don't think this change is needed.


const resizeObserver = this._engine.onResizeObservable.add(() => {
updateMaterial();
Expand All @@ -1720,6 +1719,10 @@ export class Viewer implements IDisposable {
};
}

// Remove previous meshes and materials.
high.pipeline.clearShadowCastingMeshes();
high.pipeline.clearShadowReceivingMaterials();

for (const model of this._loadedModelsBacking) {
const meshes = model.assetContainer.meshes;
for (const mesh of meshes) {
Expand All @@ -1733,10 +1736,13 @@ export class Viewer implements IDisposable {
}

high.pipeline.onVoxelizationCompleteObservable.addOnce(() => {
this._snapshotHelper.disableSnapshotRendering();
updateMaterial();
high.ground.setEnabled(true);
high.pipeline.toggleShadow(true);
high.ground.setEnabled(true);
this._snapshotHelper.enableSnapshotRendering();
this._markSceneMutated();
});

high.ground.position.y = worldBounds.extents.min[1];
Expand Down