From b7d11ce4111f21ed42461809f240be38f93ee288 Mon Sep 17 00:00:00 2001 From: Ringo Date: Fri, 22 Dec 2023 22:14:44 -0500 Subject: [PATCH] refactor code, fixed bug and added transition animation --- web/index.js | 362 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 223 insertions(+), 139 deletions(-) diff --git a/web/index.js b/web/index.js index cdc919f..71b2187 100644 --- a/web/index.js +++ b/web/index.js @@ -1,10 +1,3 @@ -/** - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - import { ARButton, RealityAccelerator } from 'ratk'; import { BoxGeometry, @@ -23,6 +16,7 @@ import { Color, Box3 } from 'three'; +import * as TWEEN from "@tweenjs/tween.js"; import { Handy } from './handy/src/Handy.js' import { Text } from 'troika-three-text'; import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; @@ -31,10 +25,19 @@ import { update, loadPose, getMatchedPoses } from './handyworks/build/esm/handy- import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; import { DRACOLoader } from 'three/addons/loaders/DRACOLoader.js'; +// Game states +const GAME_STATE = { + START: "start", + ANSWERING: "ANSWERING", + LOADING: "loading", + END: "end" +} +let gameState = GAME_STATE.START; + // Global variables for scene components let camera, scene, renderer; let controller1, controller2; -let hand1, hand2; +let leftHand, rightHand; let controllerGrip1, controllerGrip2; let ratk; // Instance of Reality Accelerator let pendingAnchorData = null; @@ -47,12 +50,11 @@ const defaultColor = new Color(0x000000) // anchor model for the scene let anchorModel, anchorText; -let text1, group1; let questions = []; let correctAnswer = null; let userCorrectAnswerInterval = 0; -const correctIntervalThreshold = 3000; +const correctIntervalThreshold = 1500; const incorrectAnswerMultipler = 1.5; const loader = new GLTFLoader(); @@ -75,23 +77,23 @@ animate(); * Initializes the scene, camera, renderer, lighting, and AR functionalities. */ function init() { + gameState = GAME_STATE.START; scene = new Scene(); - // setupGroups() setupCamera(); setupLighting(); setupRenderer(); setupARButton(); setupController(); + setupHandy(); window.addEventListener('resize', onWindowResize); setupRATK(); - } let questionIndex = 0; // Set up the anchor -function setupAnchor(anchor) { +function setupQuestionAnchor(anchor) { console.log("anchor is being loaded...") @@ -104,6 +106,8 @@ function setupAnchor(anchor) { group.position.set(0, 0, -2); scene.add(group); + gameState = GAME_STATE.LOADING; + fetch('./question.json') .then(response => response.json()) .then(data => { @@ -129,7 +133,7 @@ function setupCamera() { camera.lookAt(new Vector3(0, 0, 0)) } -function setupQuestion(data) { +async function setupQuestion(data) { // Read the Data // Replace the answer in the question with _ @@ -140,11 +144,16 @@ function setupQuestion(data) { const loader = new GLTFLoader(); console.log("loading...", data.model) - if(anchorModel.children.length > 0) { + gameState = GAME_STATE.LOADING; + if (anchorModel.children.length > 0) { + // play animation + await animateOut(anchorModel) anchorModel.remove(anchorModel.children[0]) anchorText.text = "Loading..." anchorText.sync() } + // TODO: instead of loading up the question one by one, + // We should load all the question at once to speed things up? loader.load(`./${data.model}`, function (gltf) { const question = data.word.replace(data.answer, "_") correctAnswer = data.answer @@ -172,8 +181,10 @@ function setupQuestion(data) { anchorText.position.set(0, (aabb.max.y - aabb.min.y) * scale, -1) anchorModel.add(model); - - listenPose = true + animateIn(anchorModel).then(() => { + listenPose = true + gameState = GAME_STATE.ANSWERING; + }) }); // console.log("anchor is loaded!") @@ -257,28 +268,28 @@ function setupController() { // controllerGrip1.add(controllerModelFactory.createControllerModel(controllerGrip1)); scene.add(controllerGrip1); - hand1 = renderer.xr.getHand(0); - hand1.userData.currentHandModel = 0; + leftHand = renderer.xr.getHand(0); + leftHand.userData.currentHandModel = 0; - scene.add(hand1); + scene.add(leftHand); handModels.left = [ - handModelFactory.createHandModel(hand1, 'mesh'), - handModelFactory.createHandModel(hand1, 'spheres'), - handModelFactory.createHandModel(hand1, 'boxes') + handModelFactory.createHandModel(leftHand, 'mesh'), + handModelFactory.createHandModel(leftHand, 'spheres'), + handModelFactory.createHandModel(leftHand, 'boxes') ]; handModels.left[0].visible = true; handModels.left[1].frustumCulled = false; - hand1.add(handModels.left[0]); + leftHand.add(handModels.left[0]); // console.log(hand1); - hand1.addEventListener('pinchend', function () { + leftHand.addEventListener('pinchend', function () { if (anchorCreated == false) { pendingAnchorData = { - position: hand1.position.clone(), - quaternion: hand1.quaternion.clone(), + position: leftHand.position.clone(), + quaternion: leftHand.quaternion.clone(), }; anchorCreated = true; } @@ -290,52 +301,55 @@ function setupController() { // controllerGrip2.add(controllerModelFactory.createControllerModel(controllerGrip2)); scene.add(controllerGrip2); - hand2 = renderer.xr.getHand(1); - hand2.userData.currentHandModel = 0; - scene.add(hand2); + rightHand = renderer.xr.getHand(1); + rightHand.userData.currentHandModel = 0; + scene.add(rightHand); handModels.right = [ - handModelFactory.createHandModel(hand2, 'mesh'), - handModelFactory.createHandModel(hand2, 'spheres'), - handModelFactory.createHandModel(hand2, 'boxes') + handModelFactory.createHandModel(rightHand, 'mesh'), + handModelFactory.createHandModel(rightHand, 'spheres'), + handModelFactory.createHandModel(rightHand, 'boxes') ]; for (let i = 0; i < 3; i++) { const model = handModels.right[i]; model.visible = i == 0; - hand2.add(model); + rightHand.add(model); } - hand2.addEventListener('pinchend', function () { + rightHand.addEventListener('pinchend', function () { if (anchorCreated == false) { pendingAnchorData = { - position: hand1.position.clone(), - quaternion: hand1.quaternion.clone(), + position: leftHand.position.clone(), + quaternion: leftHand.quaternion.clone(), }; anchorCreated = true; } }); +} - Handy.makeHandy(hand1) - Handy.makeHandy(hand2) +function setupHandy() { + // once we make handy + Handy.makeHandy(leftHand) + Handy.makeHandy(rightHand) if (handyLeft == null) { handyLeft = Handy.hands.getLeft() - if (handyLeft) { - console.log("found left!") - handyLeft.addEventListener("pose changed", (event) => { - // console.log(event.message) - }) - } + // if (handyLeft) { + // console.log("found left!") + // handyLeft.addEventListener("pose changed", (event) => { + // // console.log(event.message) + // }) + // } } - if (handyRight) { + if (handyRight == null) { handyRight = Handy.hands.getRight() - if (handyRight) { - console.log("found right!") - handyRight.addEventListener("pose changed", (event) => { - // console.log(event.message) - }) - } + // if (handyRight) { + // console.log("found right!") + // handyRight.addEventListener("pose changed", (event) => { + // // console.log(event.message) + // }) + // } } } @@ -416,41 +430,6 @@ function setupRATK() { }); } -function setupGroups(anchor) { - group1 = new Group() - text1 = new Text() - text1.text = 'L_MP' - // text1.position.set(-2, 2, -2) - text1.anchorX = 'center'; - text1.anchorY = 'bottom'; - text1.frustumCulled = false // always render - text1.fontSize = 1 - - group1.add(text1) - - const loader = new GLTFLoader(); - loader.load('/models/lamp.glb', function (gltf) { - const model = gltf.scene; - // Get bounding box of the model - model.scale.set(1, 1, 1) - model.frustumCulled = false // always render - model.position.set(0, 0, -2) - const aabb = new Box3(); - aabb.setFromObject(model); - const height = (aabb.max.y - aabb.min.y) / 2; - model.position.y = height; - group1.add(model); - - text1.position.set(0, (aabb.max.y - aabb.min.y), -2) - }); - // TODO: should be directly infront of user - group1.position.set(0, 0, -2); - scene.add(group1) - - // start listening for the pose - listenPose = true -} - /** * Handles the addition of a new plane detected by RATK. */ @@ -510,78 +489,132 @@ let lastTime = 0; function render(arg) { const delta = arg - lastTime lastTime = arg - + TWEEN.update(); handlePendingAnchors(); ratk.update(); updateSemanticLabels(); const session = renderer.xr.getSession(); - // if (session) { - // const frame = renderer.xr.getFrame(); - // const referenceSpace = renderer.xr.getReferenceSpace(); - // const xrInputSources = session.inputSources; - // update(xrInputSources, referenceSpace, frame, (arg) => { - // console.log(arg); - // }); - // } + // TODO: there are some logic here that we use to setup the hands + // maybe we can move this to a separate function // Let's disable left hand input for now // LEFT HAND - if (handyLeft == null) { - handyLeft = Handy.hands.getLeft() - // if(handyLeft) { - // console.log("left hand event registered") - // handyLeft.addEventListener("pose changed", (event) => { - // // console.log(event.message) - // }) - // } - } else if (listenPose) { - // if(handyLeft.isPose('asl a', 3000)) { - // handyLeft.traverse((child) => { if(child.material) child.material.color = new Color( "green" ) }) - // } else { - // handyLeft.traverse((child) => { if(child.material) child.material.color = new Color("gray") }) - // } - } + // if (handyLeft == null) { + // handyLeft = Handy.hands.getLeft() + // // if(handyLeft) { + // // console.log("left hand event registered") + // // handyLeft.addEventListener("pose changed", (event) => { + // // // console.log(event.message) + // // }) + // // } + // } else if (listenPose) { + // // if(handyLeft.isPose('asl a', 3000)) { + // // handyLeft.traverse((child) => { if(child.material) child.material.color = new Color( "green" ) }) + // // } else { + // // handyLeft.traverse((child) => { if(child.material) child.material.color = new Color("gray") }) + // // } + // } // RIGHT HAND + // if (handyRight == null) { + // handyRight = Handy.hands.getRight() + // if (handyRight) { + // console.log("right hand event registered") + // handyRight.addEventListener("pose changed", (event) => { + // console.log(event.message) + // }) + // } + // } else if (listenPose) { + // if (rightHand.isPose(`asl ${correctAnswer?.toLowerCase()}`, 3000)) { + // // Get the time away from last frame in threejs + // userCorrectAnswerInterval += lastTime + // if (userCorrectAnswerInterval > correctIntervalThreshold) { + // // user has held the pose for 3 seconds + // // transition to the next state + // // Play success audio + // listenPose = false + // userCorrectAnswerInterval = correctIntervalThreshold; + // anchorText.text = anchorText.text.replace("_", correctAnswer) + // anchorText.sync() + // // next question + // setTimeout(() => { + // questionIndex += 1 + // setupQuestion(questions[questionIndex]) + // }, 3000); + // } + // // if time reached 3 seconds, then user has held the pose for 3 seconds + // } else { + // if (userCorrectAnswerInterval > 0) { + // const newInterval = userCorrectAnswerInterval - (lastTime * incorrectAnswerMultipler) + // userCorrectAnswerInterval = newInterval > 0 ? newInterval : 0 + // } + // } + // // console.log(userCorrectAnswerInterval) + // handyRight.traverse((child) => { if (child.material) child.material.color = new Color().lerpColors(defaultColor, successColor, userCorrectAnswerInterval / 3000) }) + // } + + if (gameState == GAME_STATE.ANSWERING) { + // here we start listening to the pose + listenRightHand(delta); + } + + Handy.update() + renderer.render(scene, camera); +} + +function listenRightHand(delta) { if (handyRight == null) { - handyRight = Handy.hands.getRight() - if (handyRight) { - console.log("right hand event registered") - handyRight.addEventListener("pose changed", (event) => { - console.log(event.message) - }) - } - } else if (listenPose) { - if (hand2.isPose(`asl ${correctAnswer?.toLowerCase()}`, 3000)) { + handyRight = Handy.hands.getRight(); + } + + if (handyRight != null) { + // Right Hand... + if (rightHand.isPose(`asl ${correctAnswer?.toLowerCase()}`, 3000)) { // Get the time away from last frame in threejs - userCorrectAnswerInterval += lastTime + userCorrectAnswerInterval += delta; if (userCorrectAnswerInterval > correctIntervalThreshold) { - // user has held the pose for 3 seconds + // user has held the pose for 1.5 seconds // transition to the next state - // Play success audio - listenPose = false - userCorrectAnswerInterval = correctIntervalThreshold; - anchorText.text = anchorText.text.replace("_", correctAnswer) - anchorText.sync() - // next question - setTimeout(() => { - questionIndex += 1 - setupQuestion(questions[questionIndex]) - }, 3000); + if (questionIndex == questions.length - 1) { + endGame(); + } else { + nextQuestion(); + } } // if time reached 3 seconds, then user has held the pose for 3 seconds } else { if (userCorrectAnswerInterval > 0) { - const newInterval = userCorrectAnswerInterval - (lastTime * incorrectAnswerMultipler) - userCorrectAnswerInterval = newInterval > 0 ? newInterval : 0 + const newInterval = userCorrectAnswerInterval - (delta * incorrectAnswerMultipler); + userCorrectAnswerInterval = newInterval > 0 ? newInterval : 0; } } - // console.log(userCorrectAnswerInterval) - handyRight.traverse((child) => { if (child.material) child.material.color = new Color().lerpColors(defaultColor, successColor, userCorrectAnswerInterval / 3000) }) + handyRight?.traverse((child) => { if (child.material) child.material.color = new Color().lerpColors(defaultColor, successColor, userCorrectAnswerInterval / correctIntervalThreshold); }); } - Handy.update() - renderer.render(scene, camera); +} + +function endGame() { + gameState = GAME_STATE.END; + + animateOut(anchorModel).then(() => { + anchorModel.remove(anchorModel.children[0]) + anchorText.text = "Thanks for Playing"; + anchorText.sync(); + }) +} + +function nextQuestion() { + listenPose = false; + gameState = GAME_STATE.LOADING; + userCorrectAnswerInterval = correctIntervalThreshold; + anchorText.text = anchorText.text.replace("_", correctAnswer); + anchorText.sync(); + //todo: play success audio + // next question + setTimeout(() => { + questionIndex += 1; + setupQuestion(questions[questionIndex]); + }, 1000); } /** @@ -596,12 +629,14 @@ function handlePendingAnchors() { true, ) .then((anchor) => { - setupAnchor(anchor) + setupQuestionAnchor(anchor) }); pendingAnchorData = null; } } +// This is from the sample project, where an anchor is created +// when the user pinches the controller function buildAnchorMarker(anchor, isRecovered) { const geometry = new BoxGeometry(0.05, 0.05, 0.05); const material = new MeshBasicMaterial({ @@ -625,3 +660,52 @@ function updateSemanticLabels() { } }); } + +function animateIn(obj) { + return new Promise((resolve) => { + var originalPosition = obj.position.y; + var originalScale = { x: 1, y: 1, z: 1 } + obj.scale.set(0, 0, 0); + + obj.position.y = originalPosition - 1; + const positionTween = new TWEEN.Tween(obj.position) + .to({ y: originalPosition }, 1000) // Move upward to y = 2 in 2000 milliseconds (2 seconds) + .easing(TWEEN.Easing.Linear.None) + .start(); + + const rotationTween = new TWEEN.Tween(obj.rotation) + .to({ y: Math.PI * 2 }, 1000) // Rotate 360 degrees in 3000 milliseconds (3 seconds) + .easing(TWEEN.Easing.Linear.None) + .start(); + + const scaleTween = new TWEEN.Tween(obj.scale) + .to(originalScale, 1000) // Scale up to (1, 1, 1) in 2000 milliseconds (2 seconds) + .easing(TWEEN.Easing.Linear.None) + .start(); + + // when scale, rotation, and positiiion are done, resolve the promise + setTimeout(() => { + resolve() + }, 1000); + }) +} + +function animateOut(obj) { + return new Promise((resolve) => { + const positionTween = new TWEEN.Tween(obj.position) + .to({ y: 0 }, 250) + .easing(TWEEN.Easing.Bounce.None) + .start(); + + // Create a Tween for scaling + const scaleTween = new TWEEN.Tween(obj.scale) + .to({ x: 0, y: 0, z: 0 }, 1000) + .easing(TWEEN.Easing.Linear.None) + .start(); + + // when scale, rotation, and positiiion are done, resolve the promise + setTimeout(() => { + resolve() + }, 1000); + }) +} \ No newline at end of file