Skip to content
This repository has been archived by the owner on Aug 21, 2024. It is now read-only.

Commit

Permalink
Reflection probes (#10458)
Browse files Browse the repository at this point in the history
* mockup

* add editor ui

* invert distance weighting

* add to component shelf

* checkpoint

* combine reflection probe system and envmap component
add "Bake Probes" button
add resource management and proper cleanup
add single dedicated webgl renderer

* localize "bake probes" button

* localize "bake probes" button
  • Loading branch information
dinomut1 authored Jun 28, 2024
1 parent b6e0836 commit 3a9c39a
Show file tree
Hide file tree
Showing 10 changed files with 408 additions and 8 deletions.
8 changes: 7 additions & 1 deletion packages/client-core/i18n/en/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,8 @@
"lbl-bake": "Envmap Bake",
"lbl-textureType": "Texture Type",
"lbl-textureUrl": "Texture URL",
"lbl-intensity": "Envmap Intensity"
"lbl-intensity": "Envmap Intensity",
"bake-reflection-probes": "Bake Probes"
},
"trigger": {
"name": "Trigger",
Expand Down Expand Up @@ -761,6 +762,11 @@
"lbl-trebleMultiplier": "Treble Multiplier",
"lbl-midMultiplier": "Mid Multiplier"
},
"reflectionProbe": {
"name": "Reflection Probe",
"description": "A reflection probe that captures the environment and uses it for reflection.",
"src": "Source"
},
"renderSettings": {
"name": "Render Settings",
"description": "Customise the scene renderer settings",
Expand Down
5 changes: 4 additions & 1 deletion packages/editor/src/services/ComponentEditors.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import { ParticleSystemComponent } from '@etherealengine/engine/src/scene/compon
import { PlaylistComponent } from '@etherealengine/engine/src/scene/components/PlaylistComponent'
import { PortalComponent } from '@etherealengine/engine/src/scene/components/PortalComponent'
import { PrimitiveGeometryComponent } from '@etherealengine/engine/src/scene/components/PrimitiveGeometryComponent'
import { ReflectionProbeComponent } from '@etherealengine/engine/src/scene/components/ReflectionProbeComponent'
import { RenderSettingsComponent } from '@etherealengine/engine/src/scene/components/RenderSettingsComponent'
import { SDFComponent } from '@etherealengine/engine/src/scene/components/SDFComponent'
import { ScenePreviewCameraComponent } from '@etherealengine/engine/src/scene/components/ScenePreviewCamera'
Expand Down Expand Up @@ -109,6 +110,7 @@ import MountPointNodeEditor from '@etherealengine/ui/src/components/editor/prope
import ParticleSystemNodeEditor from '@etherealengine/ui/src/components/editor/properties/particle'
import PortalNodeEditor from '@etherealengine/ui/src/components/editor/properties/portal'
import PostProcessingSettingsEditor from '@etherealengine/ui/src/components/editor/properties/postProcessing'
import ReflectionProbeNodeEditor from '@etherealengine/ui/src/components/editor/properties/reflectionProbe'
import RenderSettingsEditor from '@etherealengine/ui/src/components/editor/properties/render'
import RigidBodyComponentEditor from '@etherealengine/ui/src/components/editor/properties/rigidBody'
import ScenePreviewCameraNodeEditor from '@etherealengine/ui/src/components/editor/properties/scene/previewCamera'
Expand Down Expand Up @@ -187,7 +189,8 @@ export const ComponentEditorsState = defineState({
[GrabbableComponent.name]: GrabbableComponentNodeEditor,
[ScreenshareTargetComponent.name]: ScreenshareTargetNodeEditor,
[TextComponent.name]: TextNodeEditor,
[LookAtComponent.name]: LookAtNodeEditor
[LookAtComponent.name]: LookAtNodeEditor,
[ReflectionProbeComponent.name]: ReflectionProbeNodeEditor
} as Record<string, EditorComponentType>
}
})
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import { NewVolumetricComponent } from '@etherealengine/engine/src/scene/compone
import { ParticleSystemComponent } from '@etherealengine/engine/src/scene/components/ParticleSystemComponent'
import { PortalComponent } from '@etherealengine/engine/src/scene/components/PortalComponent'
import { PrimitiveGeometryComponent } from '@etherealengine/engine/src/scene/components/PrimitiveGeometryComponent'
import { ReflectionProbeComponent } from '@etherealengine/engine/src/scene/components/ReflectionProbeComponent'
import { RenderSettingsComponent } from '@etherealengine/engine/src/scene/components/RenderSettingsComponent'
import { SDFComponent } from '@etherealengine/engine/src/scene/components/SDFComponent'
import { SceneDynamicLoadTagComponent } from '@etherealengine/engine/src/scene/components/SceneDynamicLoadTagComponent'
Expand Down Expand Up @@ -118,7 +119,8 @@ export const ComponentShelfCategoriesState = defineState({
ParticleSystemComponent,
EnvmapComponent,
SDFComponent,
PostProcessingComponent
PostProcessingComponent,
ReflectionProbeComponent
],
Scripting: [SystemComponent, VisualScriptComponent],
Settings: [SceneSettingsComponent, RenderSettingsComponent, MediaSettingsComponent, CameraSettingsComponent],
Expand Down
26 changes: 22 additions & 4 deletions packages/engine/src/scene/components/EnvmapComponent.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import {
} from 'three'

import { isClient } from '@etherealengine/common/src/utils/getEnvironment'
import { EntityUUID, UUIDComponent } from '@etherealengine/ecs'
import { EntityUUID, UUIDComponent, useQuery } from '@etherealengine/ecs'
import {
defineComponent,
getComponent,
Expand Down Expand Up @@ -73,7 +73,9 @@ import {
import { EnvMapSourceType, EnvMapTextureType } from '../constants/EnvMapEnum'
import { getRGBArray, loadCubeMapTexture } from '../constants/Util'
import { addError, removeError } from '../functions/ErrorFunctions'
import { createReflectionProbeRenderTarget } from '../functions/reflectionProbeFunctions'
import { EnvMapBakeComponent } from './EnvMapBakeComponent'
import { ReflectionProbeComponent } from './ReflectionProbeComponent'

const tempColor = new Color()

Expand Down Expand Up @@ -126,6 +128,8 @@ export const EnvmapComponent = defineComponent({
entity
)

const probeQuery = useQuery([ReflectionProbeComponent])

useEffect(() => {
updateEnvMapIntensity(mesh, component.envMapIntensity.value)
}, [mesh, component.envMapIntensity])
Expand Down Expand Up @@ -159,6 +163,17 @@ export const EnvmapComponent = defineComponent({
}
}, [component.type, component.envMapSourceColor])

useEffect(() => {
if (component.type.value !== EnvMapSourceType.Probes) return
if (!probeQuery.length) return
const [renderTexture, unload] = createReflectionProbeRenderTarget(entity, probeQuery)
component.envmap.set(renderTexture)
return () => {
unload()
component.envmap.set(null)
}
}, [component.type, probeQuery.length])

useEffect(() => {
if (!envMapTexture) return

Expand Down Expand Up @@ -197,7 +212,7 @@ export const EnvmapComponent = defineComponent({
}, [component.type, component.envMapSourceURL])

useEffect(() => {
if (!component.envmap.value) return
//if (!component.envmap.value) return
updateEnvMap(mesh, component.envmap.value as Texture)
}, [mesh, component.envmap])

Expand Down Expand Up @@ -248,11 +263,14 @@ export function updateEnvMap(obj: Mesh<any, any> | null, envmap: Texture | null)
if (Array.isArray(obj.material)) {
obj.material.forEach((mat: MeshStandardMaterial) => {
if (mat instanceof MeshMatcapMaterial) return
mat.envMap = envmap!
mat.envMap = envmap
mat.needsUpdate = true
})
} else {
if (obj.material instanceof MeshMatcapMaterial) return
obj.material.envMap = envmap!
const material = obj.material as MeshStandardMaterial
material.envMap = envmap
material.needsUpdate = true
}
}

Expand Down
67 changes: 67 additions & 0 deletions packages/engine/src/scene/components/ReflectionProbeComponent.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
CPAL-1.0 License
The contents of this file are subject to the Common Public Attribution License
Version 1.0. (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
The License is based on the Mozilla Public License Version 1.1, but Sections 14
and 15 have been added to cover use of software over a computer network and
provide for limited attribution for the Original Developer. In addition,
Exhibit A has been modified to be consistent with Exhibit B.
Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
The Original Code is Ethereal Engine.
The Original Developer is the Initial Developer. The Initial Developer of the
Original Code is the Ethereal Engine team.
All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023
Ethereal Engine. All Rights Reserved.
*/

import { defineComponent, useComponent, useEntityContext } from '@etherealengine/ecs'
import { useEffect } from 'react'
import { Texture } from 'three'
import { useTexture } from '../../assets/functions/resourceLoaderHooks'
import { addError } from '../functions/ErrorFunctions'

export const ReflectionProbeComponent = defineComponent({
name: 'ReflectionProbeComponent',
jsonID: 'IR_reflectionProbe',
onInit: () => ({
src: '',
// internal
texture: null as Texture | null
}),
toJSON: (entity, component) => ({
src: component.src.value
}),
onSet: (entity, component, json) => {
if (typeof json === 'undefined') return
if (typeof json.src === 'string') {
component.src.set(json.src)
}
},
errors: ['LOADING_ERROR'],
reactor: () => {
const entity = useEntityContext()
const probeComponent = useComponent(entity, ReflectionProbeComponent)

const [probeTexture, error] = useTexture(probeComponent.src.value, entity)

useEffect(() => {
if (!probeTexture) return
probeComponent.texture.set(probeTexture)
}, [probeTexture])

useEffect(() => {
if (!error) return
probeComponent.texture.set(null)
addError(entity, ReflectionProbeComponent, 'LOADING_ERROR', 'Failed to load reflection probe texture.')
})
}
})
1 change: 1 addition & 0 deletions packages/engine/src/scene/constants/EnvMapEnum.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export const EnvMapSourceType = {
Default: 'Default' as const,
Texture: 'Texture' as const,
Color: 'Color' as const,
Probes: 'Probes' as const,
None: 'None' as const
}

Expand Down
175 changes: 175 additions & 0 deletions packages/engine/src/scene/functions/reflectionProbeFunctions.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*
CPAL-1.0 License
The contents of this file are subject to the Common Public Attribution License
Version 1.0. (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
The License is based on the Mozilla Public License Version 1.1, but Sections 14
and 15 have been added to cover use of software over a computer network and
provide for limited attribution for the Original Developer. In addition,
Exhibit A has been modified to be consistent with Exhibit B.
Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
The Original Code is Ethereal Engine.
The Original Developer is the Initial Developer. The Initial Developer of the
Original Code is the Ethereal Engine team.
All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023
Ethereal Engine. All Rights Reserved.
*/

import { Entity, getComponent } from '@etherealengine/ecs'
import { TransformComponent } from '@etherealengine/spatial'
import { createDisposable } from '@etherealengine/spatial/src/resources/resourceHooks'
import {
CanvasTexture,
EquirectangularReflectionMapping,
LinearFilter,
Mesh,
OrthographicCamera,
PlaneGeometry,
RepeatWrapping,
SRGBColorSpace,
Scene,
ShaderMaterial,
Texture,
Uniform,
WebGLRenderer
} from 'three'
import { ReflectionProbeComponent } from '../components/ReflectionProbeComponent'

let textureIndex = 0
let renderer: WebGLRenderer | null = null
let canvas: HTMLCanvasElement | null = null

export function createReflectionProbeRenderTarget(entity: Entity, probes: Entity[]): [Texture, () => void] {
if (!canvas) {
canvas = document.createElementNS('http://www.w3.org/1999/xhtml', 'canvas') as HTMLCanvasElement
canvas.style.display = 'block'
}
if (!renderer) {
renderer = new WebGLRenderer({ canvas })
}
const scene = new Scene()
const camera = new OrthographicCamera(-1, 1, 1, -1, 0.1, 10)
camera.position.z = 1

const vertexShader = `
varying vec2 vUv;
void main() {
vUv = uv;
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
}
`

const fragmentShader = `
${probes.map((probe, index) => `uniform sampler2D envMap${index};`).join('\n')}
${probes.map((probe, index) => `uniform mat4 envMapTransform${index};`).join('\n')}
uniform mat4 targetTransform;
varying vec2 vUv;
//uniform vec3 targetPosition;
float EPSILON = 0.0;
void main() {
vec3 targetPosition = targetTransform[3].xyz; // Extract position component from targetMatrix
vec3 color = vec3(0.0);
float totalWeight = 0.0;
float currentDistance = 0.0;
float weight = 0.0;
${probes
.map(
(probe, index) => `
vec3 envMapPosition${index} = envMapTransform${index}[3].xyz; // Extract position component from envMapTransforms
currentDistance = length(targetPosition - envMapPosition${index}) + EPSILON;
weight = 1.0 / currentDistance;
totalWeight += weight;
color += texture2D(envMap${index}, vUv).rgb * weight;
`
)
.join('\n')}
color /= totalWeight; // Normalize the accumulated color by the total weight
gl_FragColor = vec4(color, 1.0);
}
`

const uniforms = {
targetTransform: { value: getComponent(entity, TransformComponent).matrixWorld }
} as Record<string, any>

let index = 0
for (let i = 0; i < probes.length; i++) {
const probeComponent = getComponent(probes[i], ReflectionProbeComponent)
const transformComponent = getComponent(probes[i], TransformComponent)
if (!probeComponent.texture) continue
uniforms[`envMap${index}`] = new Uniform(probeComponent.texture)
uniforms[`envMapTransform${index}`] = { value: transformComponent.matrixWorld }
index++
}

const material = new ShaderMaterial({
vertexShader,
fragmentShader,
uniforms
})

const quad = new Mesh(new PlaneGeometry(2, 2), material)
scene.add(quad)

renderer.setSize(256, 256)
renderer.render(scene, camera)
const dupeCanvas = document.createElementNS('http://www.w3.org/1999/xhtml', 'canvas') as HTMLCanvasElement
dupeCanvas.width = 256
dupeCanvas.height = 256
const ctx = dupeCanvas.getContext('2d')
if (ctx) {
ctx.drawImage(canvas, 0, 0)
}
const [result, unload] = createDisposable(
CanvasTexture,
entity,
dupeCanvas,
EquirectangularReflectionMapping,
RepeatWrapping,
RepeatWrapping,
LinearFilter,
LinearFilter
)
result.colorSpace = SRGBColorSpace
result.needsUpdate = true

// const testMat = new MeshBasicMaterial({ map: result })
// const testQuad = new Mesh(new PlaneGeometry(1, 1), testMat)

// const testEntity = createEntity()
// setComponent(testEntity, EntityTreeComponent, {
// parentEntity: getComponent(getState(EngineState).viewerEntity, SceneComponent).children[0]
// })
// setComponent(testEntity, TransformComponent, { position: new Vector3(0, randFloat(5, 15), 0) })

// setComponent(testEntity, MeshComponent, testQuad)
// addObjectToGroup(testEntity, testQuad)
// proxifyParentChildRelationships(testQuad)
// setComponent(testEntity, NameComponent, 'Test Entity')
// setComponent(testEntity, VisibleComponent, true)
// setComponent(entity, UpdatableComponent)
// setCallback(entity, UpdatableCallback, () => {
// renderer.clear()
// renderer.render(scene, camera)
// result.needsUpdate = true
// })
result.name = `ReflectionProbeTexture__${textureIndex++}`
const fullUnload = () => {
unload()
scene.clear()
quad.geometry.dispose()
dupeCanvas.remove()
}
return [result, fullUnload]
}
Loading

0 comments on commit 3a9c39a

Please sign in to comment.