Fix #1136 markers: visibility of big imageLayer & videoLayer

packages/core/src/services/Renderer.ts

@@ -1,18 +1,21 @@
 import {
+    Box3,
     Euler,
+    Frustum,
     Group,
     Intersection,
+    LinearSRGBColorSpace,
+    Matrix4,
     Mesh,
     Object3D,
     PerspectiveCamera,
     Raycaster,
-    Renderer as ThreeRenderer,
     Scene,
+    Renderer as ThreeRenderer,
     Vector2,
-    WebGLRenderer,
-    WebGLRenderTarget,
-    LinearSRGBColorSpace,
     Vector3,
+    WebGLRenderTarget,
+    WebGLRenderer,
 } from 'three';
 import { SPHERE_RADIUS, VIEWER_DATA } from '../data/constants';
 import { SYSTEM } from '../data/system';
@@ -32,6 +35,8 @@ import { AbstractService } from './AbstractService';
 import type { AbstractAdapter } from '../adapters/AbstractAdapter';
 
 const vector2 = new Vector2();
+const matrix4 = new Matrix4();
+const box3 = new Box3();
 
 export type CustomRenderer = Pick<ThreeRenderer, 'render'> & {
     getIntersections?(raycaster: Raycaster, vector: Vector2): Array<Intersection<Mesh>>;
@@ -48,9 +53,11 @@ export class Renderer extends AbstractService {
     private readonly mesh: Mesh;
     private readonly meshContainer: Group;
     private readonly raycaster: Raycaster;
+    private readonly frustum: Frustum;
     private readonly container: HTMLElement;
 
     private timestamp?: number;
+    private frustumNeedsUpdate = true;
     private customRenderer?: CustomRenderer;
 
     get panoramaPose(): Euler {
@@ -86,6 +93,7 @@ export class Renderer extends AbstractService {
         this.scene.add(this.meshContainer);
 
         this.raycaster = new Raycaster();
+        this.frustum = new Frustum();
 
         this.container = document.createElement('div');
         this.container.className = 'psv-canvas-container';
@@ -183,6 +191,7 @@ export class Renderer extends AbstractService {
         this.camera.aspect = this.state.aspect;
         this.camera.updateProjectionMatrix();
         this.viewer.needsUpdate();
+        this.frustumNeedsUpdate = true;
     }
 
     /**
@@ -192,6 +201,7 @@ export class Renderer extends AbstractService {
         this.camera.fov = this.state.vFov;
         this.camera.updateProjectionMatrix();
         this.viewer.needsUpdate();
+        this.frustumNeedsUpdate = true;
     }
 
     /**
@@ -208,6 +218,7 @@ export class Renderer extends AbstractService {
         }
         this.camera.updateMatrixWorld();
         this.viewer.needsUpdate();
+        this.frustumNeedsUpdate = true;
     }
 
     /**
@@ -402,6 +413,38 @@ export class Renderer extends AbstractService {
         return intersections;
     }
 
+    /**
+     * Checks if an object/point is currently visible
+     */
+    isObjectVisible(value: Object3D | Vector3): boolean {
+        if (!value) {
+            return false;
+        }
+
+        if (this.frustumNeedsUpdate) {
+            matrix4.multiplyMatrices(this.camera.projectionMatrix, this.camera.matrixWorldInverse);
+            this.frustum.setFromProjectionMatrix(matrix4);
+            this.frustumNeedsUpdate = false;
+        }
+
+        if ((value as Vector3).isVector3) {
+            return this.frustum.containsPoint(value as Vector3);
+        } else if ((value as Mesh).isMesh && (value as Mesh).geometry) {
+            // Frustum.intersectsObject uses the boundingSphere by default
+            // for better precision we prefer the boundingBox
+            const mesh = value as Mesh;
+            if (!mesh.geometry.boundingBox) {
+                mesh.geometry.computeBoundingBox();
+            }
+            box3.copy(mesh.geometry.boundingBox).applyMatrix4(mesh.matrixWorld);
+            return this.frustum.intersectsBox(box3);
+        } else if ((value as Object3D).isObject3D) {
+            return this.frustum.intersectsObject(value as Object3D);
+        } else {
+            return false;
+        }
+    }
+
     /**
      * Adds an object to the THREE scene
      */

packages/cubemap-tiles-adapter/src/CubemapTilesAdapter.ts

@@ -1,11 +1,9 @@
 import type { PanoramaPosition, Position, TextureData, Viewer } from '@photo-sphere-viewer/core';
-import { AbstractAdapter, CONSTANTS, events, PSVError, utils } from '@photo-sphere-viewer/core';
+import { AbstractAdapter, CONSTANTS, PSVError, events, utils } from '@photo-sphere-viewer/core';
 import { CubemapAdapter, CubemapData, CubemapFaces } from '@photo-sphere-viewer/cubemap-adapter';
 import {
     BoxGeometry,
     BufferAttribute,
-    Frustum,
-    Matrix4,
     Mesh,
     MeshBasicMaterial,
     Texture,
@@ -15,7 +13,7 @@ import {
 import { Queue, Task } from '../../shared/Queue';
 import { buildDebugTexture, buildErrorMaterial, createWireFrame } from '../../shared/tiles-utils';
 import { CubemapMultiTilesPanorama, CubemapTilesAdapterConfig, CubemapTilesPanorama } from './model';
-import { checkPanoramaConfig, CubemapTileConfig, getTileConfig, getTileConfigByIndex, isTopOrBottom } from './utils';
+import { CubemapTileConfig, checkPanoramaConfig, getTileConfig, getTileConfigByIndex, isTopOrBottom } from './utils';
 
 type CubemapMesh = Mesh<BoxGeometry, MeshBasicMaterial[]>;
 type CubemapTexture = TextureData<Texture[], CubemapTilesPanorama | CubemapMultiTilesPanorama, CubemapData>;
@@ -68,8 +66,6 @@ const getConfig = utils.getConfigParser<CubemapTilesAdapterConfig>(
     }
 );
 
-const frustum = new Frustum();
-const projScreenMatrix = new Matrix4();
 const vertexPosition = new Vector3();
 
 /**
@@ -275,10 +271,6 @@ export class CubemapTilesAdapter extends AbstractAdapter<
             return;
         }
 
-        const camera = this.viewer.renderer.camera;
-        projScreenMatrix.multiplyMatrices(camera.projectionMatrix, camera.matrixWorldInverse);
-        frustum.setFromProjectionMatrix(projScreenMatrix);
-
         const panorama: CubemapTilesPanorama | CubemapMultiTilesPanorama = this.viewer.state.textureData.panorama;
         const panoData: CubemapData = this.viewer.state.textureData.panoData;
         const zoomLevel = this.viewer.getZoomLevel();
@@ -291,7 +283,7 @@ export class CubemapTilesAdapter extends AbstractAdapter<
             vertexPosition.fromBufferAttribute(verticesPosition, i);
             vertexPosition.applyEuler(this.viewer.renderer.sphereCorrection);
 
-            if (frustum.containsPoint(vertexPosition)) {
+            if (this.viewer.renderer.isObjectVisible(vertexPosition)) {
                 const face = Math.floor(i / NB_VERTICES_BY_PLANE);
 
                 // compute position of the segment (6 vertices)

packages/equirectangular-tiles-adapter/src/EquirectangularTilesAdapter.ts

@@ -1,10 +1,8 @@
 import type { PanoData, PanoramaPosition, Position, TextureData, Viewer } from '@photo-sphere-viewer/core';
-import { AbstractAdapter, CONSTANTS, EquirectangularAdapter, events, PSVError, utils } from '@photo-sphere-viewer/core';
+import { AbstractAdapter, CONSTANTS, EquirectangularAdapter, PSVError, events, utils } from '@photo-sphere-viewer/core';
 import {
     BufferAttribute,
-    Frustum,
     MathUtils,
-    Matrix4,
     Mesh,
     MeshBasicMaterial,
     SphereGeometry,
@@ -18,7 +16,7 @@ import {
     EquirectangularTilesAdapterConfig,
     EquirectangularTilesPanorama,
 } from './model';
-import { checkPanoramaConfig, getTileConfig, EquirectangularTileConfig, getTileConfigByIndex } from './utils';
+import { EquirectangularTileConfig, checkPanoramaConfig, getTileConfig, getTileConfigByIndex } from './utils';
 
 /* the faces of the top and bottom rows are made of a single triangle (3 vertices)
  * all other faces are made of two triangles (6 vertices)
@@ -99,8 +97,6 @@ const getConfig = utils.getConfigParser<EquirectangularTilesAdapterConfig>(
     }
 );
 
-const frustum = new Frustum();
-const projScreenMatrix = new Matrix4();
 const vertexPosition = new Vector3();
 
 /**
@@ -336,10 +332,6 @@ export class EquirectangularTilesAdapter extends AbstractAdapter<
             return;
         }
 
-        const camera = this.viewer.renderer.camera;
-        projScreenMatrix.multiplyMatrices(camera.projectionMatrix, camera.matrixWorldInverse);
-        frustum.setFromProjectionMatrix(projScreenMatrix);
-
         const panorama: EquirectangularTilesPanorama | EquirectangularMultiTilesPanorama = this.viewer.config.panorama;
         const zoomLevel = this.viewer.getZoomLevel();
         const tileConfig = getTileConfig(panorama, zoomLevel, this);
@@ -351,7 +343,7 @@ export class EquirectangularTilesAdapter extends AbstractAdapter<
             vertexPosition.fromBufferAttribute(verticesPosition, i);
             vertexPosition.applyEuler(this.viewer.renderer.sphereCorrection);
 
-            if (frustum.containsPoint(vertexPosition)) {
+            if (this.viewer.renderer.isObjectVisible(vertexPosition)) {
                 // compute position of the segment (3 or 6 vertices)
                 let segmentIndex;
                 if (i < this.SPHERE_SEGMENTS * NB_VERTICES_BY_SMALL_FACE) {

packages/markers-plugin/src/Marker.ts

@@ -69,14 +69,13 @@ export class Marker {
         return !this.is3d() ? this.element : null;
     }
 
-    get threeElement(): Object3D {
+    get threeElement(): Mesh<BufferGeometry, MeshBasicMaterial> {
         return this.is3d() ? this.element : null;
     }
 
     get video(): HTMLVideoElement {
         if (this.type === MarkerType.videoLayer) {
-            const mesh = this.threeElement.children[0] as Mesh<BufferGeometry, MeshBasicMaterial>;
-            return mesh.material.map.image;
+            return this.threeElement.material.map.image;
         } else {
             utils.logWarn(`Marker ${this.id} is not a video marker`);
         }
@@ -148,7 +147,7 @@ export class Marker {
         this.hideTooltip();
 
         if (this.is3d()) {
-            delete this.threeElement.children[0].userData[MARKER_DATA];
+            delete this.threeElement.userData[MARKER_DATA];
         } else {
             delete this.element[MARKER_DATA];
         }
@@ -655,8 +654,8 @@ export class Marker {
      * Updates a 3D marker
      */
     private __update3d() {
-        const element = this.threeElement;
-        const mesh = element.children[0] as Mesh<BufferGeometry, MeshBasicMaterial>;
+        const mesh = this.threeElement;
+        const group = mesh.parent;
 
         this.state.dynamicSize = false;
 
@@ -669,24 +668,24 @@ export class Marker {
             this.state.size = this.config.size;
 
             mesh.position.set(0.5 - this.state.anchor.x, this.state.anchor.y - 0.5, 0);
-            this.viewer.dataHelper.sphericalCoordsToVector3(this.state.position, element.position);
+            this.viewer.dataHelper.sphericalCoordsToVector3(this.state.position, group.position);
 
-            element.lookAt(0, element.position.y, 0);
+            group.lookAt(0, group.position.y, 0);
             switch (this.config.orientation) {
                 case 'horizontal':
-                    element.rotateX(this.state.position.pitch < 0 ? -Math.PI / 2 : Math.PI / 2);
+                    group.rotateX(this.state.position.pitch < 0 ? -Math.PI / 2 : Math.PI / 2);
                     break;
                 case 'vertical-left':
-                    element.rotateY(-Math.PI * 0.4);
+                    group.rotateY(-Math.PI * 0.4);
                     break;
                 case 'vertical-right':
-                    element.rotateY(Math.PI * 0.4);
+                    group.rotateY(Math.PI * 0.4);
                     break;
                 // no default
             }
 
             // 100 is magic number that gives a coherent size at default zoom level
-            element.scale.set(this.config.size.width / 100, this.config.size.height / 100, 1);
+            group.scale.set(this.config.size.width / 100, this.config.size.height / 100, 1);
 
             const p = mesh.geometry.getAttribute('position');
             this.state.positions3D = [0, 1, 3, 2].map((i) => {
@@ -780,6 +779,7 @@ export class Marker {
 
         mesh.material.opacity = this.config.opacity;
         mesh.renderOrder = 1000 + this.config.zIndex;
+        mesh.geometry.boundingBox = null; // reset box for Renderer.isObjectVisible
     }
 
     /**
@@ -845,11 +845,11 @@ export class Marker {
         const geometry = new PlaneGeometry(1, 1);
         const mesh = new Mesh(geometry, material);
         mesh.userData = { [MARKER_DATA]: this };
-        const element = new Group().add(mesh);
+        const group = new Group().add(mesh);
 
         // overwrite the visible property to be tied to the Marker instance
         // and do it without context bleed
-        Object.defineProperty(element, 'visible', {
+        Object.defineProperty(group, 'visible', {
             enumerable: true,
             get: function (this: Object3D) {
                 return (this.children[0].userData[MARKER_DATA] as Marker).state.visible;
@@ -859,7 +859,7 @@ export class Marker {
             },
         });
 
-        return element;
+        return mesh;
     }
 
     /**

packages/markers-plugin/src/MarkersPlugin.ts

@@ -298,7 +298,7 @@ export class MarkersPlugin extends AbstractConfigurablePlugin<
         if (marker.isPoly()) {
             this.svgContainer.appendChild(marker.domElement);
         } else if (marker.is3d()) {
-            this.viewer.renderer.addObject(marker.threeElement);
+            this.viewer.renderer.addObject(marker.threeElement.parent);
         } else {
             this.container.appendChild(marker.domElement);
         }
@@ -363,7 +363,7 @@ export class MarkersPlugin extends AbstractConfigurablePlugin<
         if (marker.isPoly()) {
             this.svgContainer.removeChild(marker.domElement);
         } else if (marker.is3d()) {
-            this.viewer.renderer.removeObject(marker.threeElement);
+            this.viewer.renderer.removeObject(marker.threeElement.parent);
         } else {
             this.container.removeChild(marker.domElement);
         }
@@ -678,12 +678,12 @@ export class MarkersPlugin extends AbstractConfigurablePlugin<
     }
 
     /**
-     * Determines if a point marker is visible<br>
+     * Determines if a marker is visible<br>
      * It tests if the point is in the general direction of the camera, then check if it's in the viewport
      */
     private __isMarkerVisible(marker: Marker, position: Point): boolean {
         if (marker.is3d()) {
-            return marker.state.positions3D.some((vector) => this.viewer.dataHelper.isPointVisible(vector));
+            return this.viewer.renderer.isObjectVisible(marker.threeElement);
         } else {
             return (
                 marker.state.positions3D[0].dot(this.viewer.state.direction) > 0
@@ -877,9 +877,9 @@ export class MarkersPlugin extends AbstractConfigurablePlugin<
             if (this.markers[marker.id]) {
                 if (marker.config.tooltip?.trigger === 'click') {
                     if (marker.tooltip) {
-                        this.hideMarkerTooltip(marker);
+                        this.hideMarkerTooltip(marker.id);
                     } else {
-                        this.showMarkerTooltip(marker);
+                        this.showMarkerTooltip(marker.id);
                     }
                 } else {
                     this.showMarkerPanel(marker.id);