diff --git a/examples/acgan/acganGen.html b/examples/acgan/acganGen.html index c0be5114..c96b0618 100644 --- a/examples/acgan/acganGen.html +++ b/examples/acgan/acganGen.html @@ -127,6 +127,7 @@ model.init(function() { + $("#loadingPad").hide(); }); diff --git a/src/assets/image/Plus.js b/src/assets/image/Plus.js new file mode 100644 index 00000000..4f257d7c --- /dev/null +++ b/src/assets/image/Plus.js @@ -0,0 +1,5 @@ +let PlusData = (function(){ + return "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAAAAXNSR0IArs4c6QAAAjJpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjEyODwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xMjg8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpDb2xvclNwYWNlPjE8L2V4aWY6Q29sb3JTcGFjZT4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CqDLZ4cAAAWCSURBVHgB7Z3vUeMwEMUJc9+BDqAC6ABKSCtUABUAHdACLdABVEA6gArgHqe5HY/krCx7N8pmnj8wsna9f37PSgwX5VY/Pz9HPPoROO6Xmpl/CVCAzvcBBaAAnQl0Ts8VQAE6E+icniuAAnQm0Dk9VwAF6Eygc3quAArQmUDn9FwBFKAzgc7puQIoQGcCndPHWwHf39/bmCmmbZd0n1/F+hcxVLtarRRqVQfl2i6mYCsA9NMdk903MqnL0wWxnjTYCtCbiWj9E7Fo1LzZbF5eXt7f3zG+vLxcr9fn5+che8HiDXfc39+XrDEZrhEUfBSu6MfHx5J+moEpXDvx3gPOzs6+vr5GNTg9Pf38/Bw17e1ksKeg19fXbfSBGKa3t7e9ZT1aWDABfl80tx94BuUK2I7HxzJ88Ic88ouCTzb7qMFWQAmgXBNDSUr/fZsJL8AQaCz0qfKDEqBcDUN59nN8UALsJ2K9Kgqg83G3UgB3xHoCCqDzcbdSAHfEegIKoPNxt1IAd8R6Agqg83G3UgB3xHoCCqDzcbdSAHfEegIKoPNxt1IAd8R6Agqg83G3UgB3xHoCCqDzcbdSAHfEegIKoPNxt1IAd8R6Agqg83G3UgB3xHoCCqDzcbdSAHfEegIKoPNxt1IAd8R6Agqg83G3UgB3xHqCRZv00sfBJQF2T6Rx+oy4+SfFETDtypOM5QAOx8fHth8SLdu5vr6W1BkEmZ86wPXzDmxLx4X4iePp6Qnbg6amNPIDlxRJBkaB62HQLFpOvScI8xjiKoNNerJnESASi/RT6PybtvlRZ3N0ZJPpfxTJiAmM03SaNNmXuVSAj48PKXFng8QiSzc6mfmYn6L92fd+unDpmzB2S2ddCQgZZA6zTyXg78otDpkUt8Jl5oQElIEEKtsX08TBUgHSrsRhZQJCBhNLqboh4DDRqD8cPPKmXMPIqZLlmzLnCDCso6xslMvhTZYQ5vU4R4DhbXh1dTUv8byrpG3UIGUMx+IwL37rVQbto+IlR/YmLFBaO4ninzW4/E146VMQxMPTWLoHU3FZiVHITqlTGkyDPo+h+O0jWzGYeXh4ODk5QQ+u9CcGn+g2hXjpk4Kj2efnZ0Ehg4zMlFPLZwY8EuCbAlAiEsvPsofZMwiLvzTc3t6m4FmcNIkvTMHXB2GcWZecDtvBGN8WMnzpT9bZ8Rf9LSjLmspaWFAWMzsFWRxIkc3jNOVFDTc3N6XVZGa0NdSzJPicpyA938KC9OCwjtJPVymmatgpDmVryzPaCzClk4PxKSVpbY0CtBIz9qcAxkBbw1GAVmLG/hTAGGhrOArQSszYnwIYA20NRwFaiRn7UwBjoK3hKEArMWN/CmAMtDUcBWglZuxPAYyBtoajAK3EjP0pgDHQ1nAUoJWYsT8FMAbaGo4CtBIz9qcAxkBbw1GAVmLG/hTAGGhrOArQSszYnwIYA20NRwFaiRn7hxRg+Gmc4diYzU7CRRJAPoYmAyAajiOKEUmAxFehPBRjJ7evQZJIAqR2I1JWhAomQHn7lzOxFAomwBS4pSTKDdjdFEyAktdQEqDf/TcmlCU1zYxvdmgKsWNn5b+zxc4h5f9a3XGdE9PFWwF3d3fbesMXaGwz7e88lnC4Q74eZIgVwoRrBAXHewlK0DebDb6nAdsC0QP2ha3X64uLi6EeUcZRBYjCt1pnvPcAtJRearLeRicznz08jboCfl89l+0P3RMxoq6ARB8yCEcZy0BM+zyIugIypnEXxIEIkOkR6DTkS1AgvtVSKUAVka8DBfDlW41OAaqIfB0ogC/fanQKUEXk60ABfPlWo1OAKiJfBwrgy7canQJUEfk6UABfvtXoFKCKyNeBAvjyrUanAFVEvg4UwJdvNfpfoaOhd/X3s1YAAAAASUVORK5CYII="; +})(); + +export { PlusData }; \ No newline at end of file diff --git a/src/assets/image/plus.png b/src/assets/image/plus.png new file mode 100644 index 00000000..2c74610d Binary files /dev/null and b/src/assets/image/plus.png differ diff --git a/src/elements/CloseButton.js b/src/elements/CloseButton.js index 42047c13..3518bb57 100644 --- a/src/elements/CloseButton.js +++ b/src/elements/CloseButton.js @@ -1,5 +1,5 @@ import { MinAlpha } from "../utils/Constant"; -import { CloseData } from "../assets/image/CloseData"; +import { TextureProvider } from "../utils/TextureProvider"; function CloseButton(size, unitLength, position, color) { @@ -25,7 +25,7 @@ CloseButton.prototype = { init: function() { - let texture = new THREE.TextureLoader().load( CloseData ); + let texture = new THREE.TextureLoader().load( TextureProvider.getTexture("close") ); let materialSide = new THREE.MeshBasicMaterial( { color: this.color, opacity: MinAlpha, transparent: true } ); let materialTop = new THREE.MeshBasicMaterial( { color: this.color, alphaMap: texture, transparent: true } ); diff --git a/src/elements/MergedAggregation.js b/src/elements/MergedAggregation.js new file mode 100644 index 00000000..8ba8b6aa --- /dev/null +++ b/src/elements/MergedAggregation.js @@ -0,0 +1,133 @@ +import { MinAlpha } from "../utils/Constant"; +import { FrameColor } from "../utils/Constant"; +import { colorUtils } from "../utils/ColorUtils"; +import { RenderPreprocessor } from "../utils/RenderPreprocessor"; +import { TextureProvider } from "../utils/TextureProvider"; + +function MergedAggregation(operator, width, height, actualWidth, actualHeight, depth, color) { + + this.operator = operator; + this.width = width; + this.height = height; + this.actualWidth = actualWidth; + this.actualHeight = actualHeight; + this.depth = depth; + + this.color = color; + + this.cube = undefined; + this.aggregationElement = undefined; + + this.dataArray = undefined; + this.dataTexture = undefined; + + this.dataMaterial = undefined; + this.clearMaterial = undefined; + + this.init(); + +} + +MergedAggregation.prototype = { + + init: function() { + + let amount = this.width * this.height; + let data = new Uint8Array(amount); + this.dataArray = data; + let dataTex = new THREE.DataTexture(data, this.width, this.height, THREE.LuminanceFormat, THREE.UnsignedByteType); + this.dataTexture = dataTex; + + dataTex.magFilter = THREE.NearestFilter; + dataTex.needsUpdate = true; + + let material = new THREE.MeshBasicMaterial({ color: this.color, alphaMap: dataTex, transparent: true }); + + let geometry = new THREE.BoxBufferGeometry(this.actualWidth, this.depth, this.actualHeight); + + let basicMaterial = new THREE.MeshBasicMaterial({ + color: this.color, opacity: MinAlpha, transparent: true + }); + + let materials = [ + basicMaterial, + basicMaterial, + material, + material, + basicMaterial, + basicMaterial + ]; + + this.dataMaterial = materials; + + let operatorTexture = new THREE.TextureLoader().load( TextureProvider.getTexture(this.operator) ); + let operatorMaterial = new THREE.MeshBasicMaterial( { color: this.color, alphaMap: operatorTexture, transparent: true} ); + + let clearMaterial = [ + basicMaterial, + basicMaterial, + operatorMaterial, + operatorMaterial, + basicMaterial, + basicMaterial + ]; + + this.clearMaterial = clearMaterial; + + let cube = new THREE.Mesh(geometry, materials); + + cube.position.set(0, 0, 0); + cube.elementType = "aggregationElement"; + cube.clickable = true; + cube.hoverable = true; + + this.cube = cube; + + let edgesGeometry = new THREE.EdgesGeometry(geometry); + let edgesLine = new THREE.LineSegments(edgesGeometry, new THREE.LineBasicMaterial({ + color: FrameColor + })); + + let aggregationGroup = new THREE.Object3D(); + aggregationGroup.add(cube); + aggregationGroup.add(edgesLine); + + this.aggregationElement = aggregationGroup; + + this.clear(); + }, + + getElement: function() { + return this.aggregationElement; + }, + + setLayerIndex: function(layerIndex) { + this.cube.layerIndex = layerIndex; + }, + + clear: function() { + + let zeroValue = new Int8Array(this.width * this.height); + let colors = colorUtils.getAdjustValues(zeroValue); + + this.updateVis(colors); + this.cube.material = this.clearMaterial; + + }, + + updateVis: function(colors) { + + let renderColor = RenderPreprocessor.preProcessFmColor(colors, this.width, this.height); + + for (let i = 0; i < renderColor.length; i++) { + this.dataArray[i] = renderColor[i] * 255; + } + + this.dataTexture.needsUpdate = true; + this.cube.material = this.dataMaterial; + + } + +}; + +export { MergedAggregation }; \ No newline at end of file diff --git a/src/elements/MergedFeatureMap.js b/src/elements/MergedFeatureMap.js new file mode 100644 index 00000000..a71b029b --- /dev/null +++ b/src/elements/MergedFeatureMap.js @@ -0,0 +1,240 @@ +import { MinAlpha } from "../utils/Constant"; +import { BasicMaterialOpacity } from "../utils/Constant"; +import { colorUtils } from "../utils/ColorUtils"; +import { TextHelper } from "../utils/TextHelper"; +import { TextFont } from "../assets/fonts/TextFont"; +import { RenderPreprocessor } from "../utils/RenderPreprocessor"; +import { TextureProvider } from "../utils/TextureProvider"; + +function MergedFeatureMap(operator, width, height, actualWidth, actualHeight, initCenter, color) { + + this.operator = operator; + + this.fmWidth = width; + this.fmHeight = height; + + this.actualWidth = actualWidth; + this.actualHeight = actualHeight; + this.color = color; + + this.neuralLength = width * height; + + this.unitLength = this.actualWidth / this.fmWidth; + + this.fmCenter = { + x: initCenter.x, + y: initCenter.y, + z: initCenter.z + }; + + this.dataArray = undefined; + this.dataTexture = undefined; + this.featureMap = undefined; + this.featureGroup = undefined; + + this.font = TextFont; + + this.textSize = TextHelper.calcFmTextSize(this.actualWidth); + + this.widthText = undefined; + this.heightText = undefined; + + this.dataMaterial = undefined; + this.clearMaterial = undefined; + + this.init(); + +} + +MergedFeatureMap.prototype = { + + init: function() { + + let amount = this.fmWidth * this.fmHeight; + let data = new Uint8Array(amount); + this.dataArray = data; + + let dataTex = new THREE.DataTexture(data, this.fmWidth, this.fmHeight, THREE.LuminanceFormat, THREE.UnsignedByteType); + this.dataTexture = dataTex; + + dataTex.magFilter = THREE.NearestFilter; + dataTex.needsUpdate = true; + + let boxGeometry = new THREE.BoxBufferGeometry(this.actualWidth, this.unitLength, this.actualHeight); + + let material = new THREE.MeshBasicMaterial({ color: this.color, alphaMap: dataTex, transparent: true }); + let basicMaterial = new THREE.MeshBasicMaterial({ + color: this.color, transparent: true, opacity: BasicMaterialOpacity + }); + + let materials = [ + basicMaterial, + basicMaterial, + material, + material, + basicMaterial, + basicMaterial + ]; + + this.dataMaterial = materials; + + let operatorTexture = new THREE.TextureLoader().load( TextureProvider.getTexture(this.operator) ); + let operatorMaterial = new THREE.MeshBasicMaterial( { color: this.color, alphaMap: operatorTexture, transparent: true} ); + + let clearMaterial = [ + basicMaterial, + basicMaterial, + operatorMaterial, + operatorMaterial, + basicMaterial, + basicMaterial + ]; + + this.clearMaterial = clearMaterial; + + let cube = new THREE.Mesh(boxGeometry, materials); + cube.elementType = "featureMap"; + cube.hoverable = true; + + this.featureMap = cube; + + let featureGroup = new THREE.Object3D(); + featureGroup.position.set(this.fmCenter.x, this.fmCenter.y, this.fmCenter.z); + featureGroup.add(cube); + this.featureGroup = featureGroup; + + this.clear(); + + }, + + getElement: function() { + return this.featureGroup; + }, + + updateVis: function(colors) { + + let renderColor = RenderPreprocessor.preProcessFmColor(colors, this.fmWidth, this.fmHeight); + for (let i = 0; i < renderColor.length; i++) { + this.dataArray[i] = renderColor[i] * 255; + } + this.dataTexture.needsUpdate = true; + + this.featureMap.material = this.dataMaterial; + + }, + + updatePos: function(pos) { + + this.fmCenter.x = pos.x; + this.fmCenter.y = pos.y; + this.fmCenter.z = pos.z; + this.featureGroup.position.set(pos.x, pos.y, pos.z); + + }, + + clear: function() { + + let zeroValue = new Int8Array(this.neuralLength); + + let colors = colorUtils.getAdjustValues(zeroValue); + + this.updateVis(colors); + + this.featureMap.material = this.clearMaterial; + + }, + + setLayerIndex: function(layerIndex) { + this.featureMap.layerIndex = layerIndex; + }, + + setFmIndex: function(fmIndex) { + this.featureMap.fmIndex = fmIndex; + }, + + showText: function() { + + let widthInString = this.fmWidth.toString(); + let heightInString = this.fmHeight.toString(); + + let material = new THREE.MeshBasicMaterial( { color: this.color } ); + + let widthGeometry = new THREE.TextGeometry( widthInString, { + font: this.font, + size: this.textSize, + height: Math.min(this.unitLength, 1), + curveSegments: 8, + } ); + + let widthText = new THREE.Mesh(widthGeometry, material); + + let widthTextPos = TextHelper.calcFmWidthTextPos( + widthInString.length, + this.textSize, + this.actualHeight, + { + x: this.featureMap.position.x, + y: this.featureMap.position.y, + z: this.featureMap.position.z + } + ); + + widthText.position.set( + widthTextPos.x, + widthTextPos.y, + widthTextPos.z + ); + + widthText.rotateX( - Math.PI / 2 ); + + let heightGeometry = new THREE.TextGeometry( heightInString, { + font: this.font, + size: this.textSize, + height: Math.min(this.unitLength, 1), + curveSegments: 8, + } ); + + let heightText = new THREE.Mesh(heightGeometry, material); + + let heightTextPos = TextHelper.calcFmHeightTextPos( + heightInString.length, + this.textSize, + this.actualWidth, + { + x: this.featureMap.position.x, + y: this.featureMap.position.y, + z: this.featureMap.position.z + } + ); + + heightText.position.set( + heightTextPos.x, + heightTextPos.y, + heightTextPos.z + ); + + heightText.rotateX( - Math.PI / 2 ); + + this.widthText = widthText; + this.heightText = heightText; + + this.featureGroup.add(this.widthText); + this.featureGroup.add(this.heightText); + this.isTextShown = true; + + }, + + hideText: function() { + + this.featureGroup.remove(this.widthText); + this.featureGroup.remove(this.heightText); + this.widthText = undefined; + this.heightText = undefined; + + this.isTextShown = false; + + } + +}; + +export { MergedFeatureMap }; \ No newline at end of file diff --git a/src/layer/abstract/Layer.js b/src/layer/abstract/Layer.js index 99a4716a..9918257d 100644 --- a/src/layer/abstract/Layer.js +++ b/src/layer/abstract/Layer.js @@ -71,6 +71,9 @@ function Layer(config) { // actualWidth / width this.unitLength = undefined; + // identify whether is merged layer + this.isMerged = false; + this.loadBasicLayerConfig(config); } @@ -121,10 +124,6 @@ Layer.prototype = { }, - setNextLayer: function(layer) { - this.nextLayer = layer; - }, - setLastLayer: function(layer) { this.lastLayer = layer; }, diff --git a/src/layer/abstract/Layer3d.js b/src/layer/abstract/Layer3d.js index 1683d3cd..0172669d 100644 --- a/src/layer/abstract/Layer3d.js +++ b/src/layer/abstract/Layer3d.js @@ -85,8 +85,6 @@ Layer3d.prototype = Object.assign(Object.create(Layer.prototype), { for (let i = 0; i < this.depth; i++) { - console.log(centers[i]); - let segregationHandler = new FeatureMap( this.width, this.height, diff --git a/src/layer/merge/Add.js b/src/layer/merge/Add.js index 6b1b4a9d..4ad1e5e3 100644 --- a/src/layer/merge/Add.js +++ b/src/layer/merge/Add.js @@ -1,4 +1,44 @@ -function Add() { +import { BasicLayer1d } from "../prime/BasicLayer1d"; +import { BasicLayer2d } from "../prime/BasicLayer2d"; +import { BasicLayer3d } from "../prime/BasicLayer3d"; +import {MergedLayer3d} from "./MergedLayer3d"; + +function Add(layerList) { + + let mergedElements = []; + + let depth; + + if (layerList.length > 0) { + depth = layerList[0].layerDimension; + } else { + console.error("Merge Layer missing elements."); + } + + for (let i = 0; i < layerList.length; i++) { + + if (layerList[i].layerDimension !== depth) { + console.error("Can not add layer with different depth."); + } + + mergedElements.push(layerList[i]); + } + + if (mergedElements[0].layerDimension === 1) { + return ; + } else if (mergedElements[0].layerDimension === 2) { + return new BasicLayer2d({shape: [100, 100]}); + } else if (mergedElements[0].layerDimension === 3) { + + let mergedLayer = new MergedLayer3d({ + operator: "add" + }); + mergedLayer.setMergedElements(mergedElements); + + return mergedLayer; + } else { + + } } diff --git a/src/layer/merge/MergedLayer.js b/src/layer/merge/MergedLayer.js new file mode 100644 index 00000000..667e3fde --- /dev/null +++ b/src/layer/merge/MergedLayer.js @@ -0,0 +1,215 @@ +import { CloseButton } from "../../elements/CloseButton"; +import { LineGroupGeometry } from "../../elements/LineGroupGeometry"; +import { BasicMaterialOpacity } from "../../utils/Constant"; + +function MergedLayer(config) { + + this.scene = undefined; + this.layerIndex = undefined; + this.center = undefined; + this.nextLayer = undefined; + this.lastLayer = undefined; + + // store all neural value as an array + + this.neuralValue = undefined; + + this.activation = undefined; + this.neuralNum = undefined; + this.inputShape = []; + this.outputShape = []; + this.neuralGroup = undefined; + + // output index to fit the layer + this.resourceOutputIndex = undefined; + + // color for layer neural visualization + this.color = undefined; + + // store the reference for layer aggregation + this.aggregationHandler = undefined; + + // store the reference for close button + this.closeButtonHandler = undefined; + + // center position is the left-most for layer, type: {x: value , y: value, z: value} + this.leftMostCenter = undefined; + + // actual width and height in three.js scene + this.actualWidth = undefined; + this.actualHeight = undefined; + + // actual depth for layer aggregation + this.actualDepth = undefined; + + // actualWidth / width + this.unitLength = undefined; + + // store hook between layers + this.nextHookHandler = undefined; + this.lastHookHandler = undefined; + + // store the line group system element + let lineMat = new THREE.LineBasicMaterial( { + color: 0xffffff, + opacity: BasicMaterialOpacity, + transparent:true, + vertexColors: THREE.VertexColors + } ); + let lineGeom = new THREE.Geometry(); + lineGeom.dynamic = true; + this.lineGroup = new THREE.Line(lineGeom, lineMat); + + // handler for element showing text + this.textElementHandler = undefined; + + // config for text and relation line + this.textSystem = undefined; + this.relationSystem = undefined; + + this.isOpen = undefined; + + // actualWidth / width + this.unitLength = undefined; + + // identify whether is merged layer + this.isMerged = true; + + this.operator = undefined; + + + this.loadBasicLayerConfig(config); + + +} + +MergedLayer.prototype = { + + loadBasicLayerConfig: function(config) { + + if (config !== undefined) { + + if (config.initStatus !== undefined) { + + if (config.initStatus === "open") { + this.isOpen = true; + } else if (config.initStatus === "close") { + this.isOpen = false; + } else { + console.error("\"initStatus\" property do not support for " + config.initStatus + ", use \"open\" or \"close\" instead."); + } + + } + + if (config.color !== undefined) { + this.color = config.color; + } + + if (config.name !== undefined) { + this.name = config.name; + } + + } + + }, + + loadBasicModelConfig: function(modelConfig) { + + if (this.isOpen === undefined) { + this.isOpen = modelConfig.layerInitStatus; + } + + if (this.relationSystem === undefined) { + this.relationSystem = modelConfig.relationSystem; + } + + if (this.textSystem === undefined) { + this.textSystem = modelConfig.textSystem; + } + + }, + + setMergedElements: function(mergedElements) { + + for (let i = 0; i < mergedElements.length; i++) { + this.mergedElements.push(mergedElements[i]); + } + + }, + + setEnvironment: function(scene) { + this.scene = scene; + }, + + initCloseButton: function() { + + let closeButtonPos = this.calcCloseButtonPos(); + let closeButtonSize = this.calcCloseButtonSize(); + let closeButtonHandler = new CloseButton(closeButtonSize, this.unitLength, closeButtonPos, this.color); + closeButtonHandler.setLayerIndex(this.layerIndex); + + this.closeButtonHandler = closeButtonHandler; + this.neuralGroup.add(this.closeButtonHandler.getElement()); + + }, + + disposeCloseButton: function() { + + this.neuralGroup.remove(this.closeButtonHandler.getElement()); + this.closeButtonHandler = undefined; + + }, + + getLineGroupParameters: function(selectedElement) { + + this.scene.updateMatrixWorld(); + + let lineColors = []; + let lineVertices = []; + + let relatedElements = this.getRelativeElements(selectedElement); + + let startPosition = selectedElement.getWorldPosition().sub(this.neuralGroup.getWorldPosition()); + + for (let i = 0; i < relatedElements.length; i++) { + + lineColors.push(new THREE.Color(this.color)); + lineColors.push(new THREE.Color(this.color)); + + lineVertices.push(relatedElements[i].getWorldPosition().sub(this.neuralGroup.getWorldPosition())); + lineVertices.push(startPosition); + + } + + return { + lineColors: lineColors, + lineVertices: lineVertices + } + + }, + + initLineGroup: function(selectedElement) { + + let lineGroupParameters = this.getLineGroupParameters(selectedElement); + + let lineGroupGeometryHandler = new LineGroupGeometry( + lineGroupParameters.lineVertices, + lineGroupParameters.lineColors + ); + this.lineGroup.geometry = lineGroupGeometryHandler.getElement(); + this.lineGroup.material.needsUpdate = true; + + this.neuralGroup.add(this.lineGroup); + + }, + + disposeLineGroup: function() { + + this.lineGroup.geometry.dispose(); + this.neuralGroup.remove(this.lineGroup); + + } + +}; + +export { MergedLayer }; \ No newline at end of file diff --git a/src/layer/merge/MergedLayer1d.js b/src/layer/merge/MergedLayer1d.js new file mode 100644 index 00000000..47b4a5c6 --- /dev/null +++ b/src/layer/merge/MergedLayer1d.js @@ -0,0 +1,14 @@ +function MergedLayer1d() { + + + this.isMerged = true; + +} + +MergedLayer1d.prototype = { + + + +}; + +export { MergedLayer1d }; \ No newline at end of file diff --git a/src/layer/merge/MergedLayer2d.js b/src/layer/merge/MergedLayer2d.js new file mode 100644 index 00000000..e69de29b diff --git a/src/layer/merge/MergedLayer3d.js b/src/layer/merge/MergedLayer3d.js new file mode 100644 index 00000000..5e385936 --- /dev/null +++ b/src/layer/merge/MergedLayer3d.js @@ -0,0 +1,401 @@ +import { fmCenterGenerator } from "../../utils/FmCenterGenerator"; +import {MergedLayer} from "./MergedLayer"; +import { ChannelDataGenerator } from "../../utils/ChannelDataGenerator"; +import { colorUtils } from "../../utils/ColorUtils"; +import { MapTransitionFactory } from "../../animation/MapTransitionTween"; +import { CloseButtonRatio } from "../../utils/Constant"; +import { MergedAggregation } from "../../elements/MergedAggregation"; +import { MergedFeatureMap } from "../../elements/MergedFeatureMap"; +import { MergedLayerValidator } from "../../utils/MergedLayerValidator"; +import { MergedShapeGenerator } from "../../utils/MergedShapeGenerator"; + +function MergedLayer3d(config) { + + MergedLayer.call(this, config); + + console.log("construct merged layer 3d."); + + this.width = undefined; + this.height = undefined; + this.depth = undefined; + + this.layerDimension = 3; + + this.color = 0xff0000; + + // store all layer segregation references as a list + this.segregationHandlers = []; + + // used to define close sphere size + this.openHeight = undefined; + + this.openFmCenters = []; + this.closeFmCenters = []; + + this.aggregationStrategy = undefined; + + this.mergedElements = []; + + this.layerType = "mergedLayer3d"; + + this.loadLayerConfig(config); + +} + +MergedLayer3d.prototype = Object.assign(Object.create(MergedLayer.prototype), { + + loadLayerConfig: function(layerConfig) { + + if (layerConfig !== undefined) { + if (layerConfig.operator !== undefined) { + this.operator = layerConfig.operator; + } + } + + }, + + loadModelConfig: function(modelConfig) { + + this.loadBasicModelConfig(modelConfig); + + if (this.layerShape === undefined) { + this.layerShape = modelConfig.layerShape; + } + + if (this.aggregationStrategy === undefined) { + this.aggregationStrategy = modelConfig.aggregationStrategy; + } + + }, + + assemble: function(layerIndex) { + + this.layerIndex = layerIndex; + + console.log("validate"); + + if(!MergedLayerValidator.validate(this.operator, this.mergedElements)) { + console.error("input shape is not valid for " + this.operator + " merge function."); + } + + console.log("generate"); + + this.inputShape = MergedShapeGenerator.getShape(this.operator, this.mergedElements); + + this.width = this.inputShape[0]; + this.height = this.inputShape[1]; + this.depth = this.inputShape[2]; + + this.outputShape = [this.width, this.height, this.depth]; + + this.unitLength = this.mergedElements[0].unitLength; + this.actualWidth = this.unitLength * this.width; + this.actualHeight = this.unitLength * this.height; + + for (let i = 0; i < this.depth; i++) { + let center = { + x: 0, + y: 0, + z: 0 + }; + this.closeFmCenters.push(center); + } + + this.openFmCenters = fmCenterGenerator.getFmCenters(this.layerShape, this.depth, this.actualWidth, this.actualHeight); + + this.leftMostCenter = this.openFmCenters[0]; + this.openHeight = this.actualHeight + this.openFmCenters[this.openFmCenters.length - 1].z - this.openFmCenters[0].z; + + }, + + init: function(center, actualDepth, nextHookHandler) { + + this.center = center; + this.actualDepth = actualDepth; + this.nextHookHandler = nextHookHandler; + // this.lastHookHandler = this.lastLayer.nextHookHandler; + + this.neuralGroup = new THREE.Group(); + this.neuralGroup.position.set(this.center.x, this.center.y, this.center.z); + + if (this.depth === 1) { + this.isOpen = true; + this.initSegregationElements(this.openFmCenters); + } else { + if (this.isOpen) { + + this.initSegregationElements(this.openFmCenters); + this.initCloseButton(); + + } else { + + this.initAggregationElement(); + + } + } + + this.scene.add(this.neuralGroup); + + }, + + openLayer: function () { + + if (!this.isOpen) { + + MapTransitionFactory.openLayer(this); + + } + + }, + + closeLayer: function () { + + if (this.isOpen) { + + MapTransitionFactory.closeLayer(this); + + } + + }, + + initSegregationElements: function(centers) { + + for (let i = 0; i < this.depth; i++) { + + let segregationHandler = new MergedFeatureMap( + this.operator, + this.width, + this.height, + this.actualWidth, + this.actualHeight, + centers[i], + this.color + ); + + segregationHandler.setLayerIndex(this.layerIndex); + segregationHandler.setFmIndex(i); + + this.segregationHandlers.push(segregationHandler); + + this.neuralGroup.add(segregationHandler.getElement()); + + } + + if (this.neuralValue !== undefined) { + this.updateSegregationVis(); + } + + }, + + disposeSegregationElements: function () { + + for (let i = 0; i < this.segregationHandlers.length; i++) { + let segregationHandler = this.segregationHandlers[i]; + this.neuralGroup.remove(segregationHandler.getElement()); + } + + this.segregationHandlers = []; + + }, + + initAggregationElement: function() { + + let aggregationHandler = new MergedAggregation( + this.operator, + this.width, + this.height, + this.actualWidth, + this.actualHeight, + this.actualDepth, + this.color + ); + aggregationHandler.setLayerIndex(this.layerIndex); + + this.aggregationHandler = aggregationHandler; + this.neuralGroup.add(aggregationHandler.getElement()); + + if (this.neuralValue !== undefined) { + this.updateAggregationVis(); + } + + }, + + disposeAggregationElement: function () { + + this.neuralGroup.remove(this.aggregationHandler.getElement()); + this.aggregationHandler = undefined; + + }, + + updateValue: function (value) { + + this.neuralValue = value; + + if (this.isOpen) { + this.updateSegregationVis(); + } else { + this.updateAggregationVis(); + } + }, + + updateAggregationVis: function() { + + let aggregationUpdateValue = ChannelDataGenerator.generateAggregationData(this.neuralValue, this.depth, this.aggregationStrategy); + + let colors = colorUtils.getAdjustValues(aggregationUpdateValue); + + this.aggregationHandler.updateVis(colors); + + }, + + updateSegregationVis: function() { + + let layerOutputValues = ChannelDataGenerator.generateChannelData(this.neuralValue, this.depth); + + let colors = colorUtils.getAdjustValues(layerOutputValues); + + let featureMapSize = this.width * this.height; + + for (let i = 0; i < this.depth; i++) { + + this.segregationHandlers[i].updateVis(colors.slice(i * featureMapSize, (i + 1) * featureMapSize)); + + } + + }, + + handleHoverIn: function(hoveredElement) { + + if (this.relationSystem !== undefined && this.relationSystem) { + this.initLineGroup(hoveredElement); + } + + if (this.textSystem !== undefined && this.textSystem) { + this.showText(hoveredElement); + } + + }, + + handleHoverOut: function() { + + if (this.relationSystem !== undefined && this.relationSystem) { + this.disposeLineGroup(); + } + + if (this.textSystem !== undefined && this.textSystem) { + this.hideText(); + } + + }, + + calcCloseButtonSize: function() { + return this.openHeight * CloseButtonRatio; + }, + + calcCloseButtonPos: function() { + + let leftMostCenter = this.openFmCenters[0]; + + return { + + x: leftMostCenter.x - this.actualWidth/ 2 - 30, + y: 0, + z: 0 + + }; + + }, + + clear: function() { + + if (this.neuralValue !== undefined) { + if (this.isOpen) { + for (let i = 0; i < this.segregationHandlers.length; i++) { + this.segregationHandlers[i].clear(); + } + } else { + this.aggregationHandler.clear(); + } + this.neuralValue = undefined; + } + + }, + + provideRelativeElements: function(request) { + + let relativeElements = []; + + if (request.all !== undefined && request.all) { + + if (this.isOpen) { + + for (let i = 0; i < this.segregationHandlers.length; i++) { + relativeElements.push(this.segregationHandlers[i].getElement()); + } + + } else { + + relativeElements.push(this.aggregationHandler.getElement()); + + } + + } else { + if (request.index !== undefined) { + + if (this.isOpen) { + relativeElements.push(this.segregationHandlers[request.index].getElement()); + } else { + relativeElements.push(this.aggregationHandler.getElement()); + } + + } + } + + return relativeElements; + + }, + + handleClick: function(clickedElement) { + + if (clickedElement.elementType === "aggregationElement") { + this.openLayer(); + } else if (clickedElement.elementType === "closeButton") { + this.closeLayer(); + } + + }, + + showText: function(element) { + + if (element.elementType === "featureMap") { + + let fmIndex = element.fmIndex; + this.segregationHandlers[fmIndex].showText(); + this.textElementHandler = this.segregationHandlers[fmIndex]; + + } + + }, + + hideText: function() { + + if (this.textElementHandler !== undefined) { + + this.textElementHandler.hideText(); + this.textElementHandler = undefined; + } + + }, + + // override this function to define relative element from previous layer + getRelativeElements: function(selectedElement) { + + let relativeElements = []; + + return []; + + } + +}); + +export { MergedLayer3d }; diff --git a/src/layer/prime/BasicLayer1d.js b/src/layer/prime/BasicLayer1d.js index a901ce54..de35fe39 100644 --- a/src/layer/prime/BasicLayer1d.js +++ b/src/layer/prime/BasicLayer1d.js @@ -36,10 +36,6 @@ BasicLayer1d.prototype = Object.assign(Object.create(Layer1d.prototype), { this.loadBasicModelConfig(modelConfig); - if (this.aggregationStrategy === undefined) { - this.aggregationStrategy = modelConfig.aggregationStrategy; - } - }, assemble: function(layerIndex) { diff --git a/src/tensorspace.js b/src/tensorspace.js index a9ac921f..18cf192b 100644 --- a/src/tensorspace.js +++ b/src/tensorspace.js @@ -38,6 +38,8 @@ import { PixelDense } from "./layer/pixel/PixelDense"; import { PixelReshape } from "./layer/pixel/PixelReshape"; import { PixelOutput } from "./layer/pixel/PixelOutput"; +import { Add } from "./layer/merge/Add"; + let layers = { Input1d: Input1d, Input2d: Input2d, @@ -82,4 +84,4 @@ let model = { PixelSequential: PixelSequential }; -export {model, layers}; \ No newline at end of file +export {model, layers, Add}; \ No newline at end of file diff --git a/src/utils/MergedLayerValidator.js b/src/utils/MergedLayerValidator.js new file mode 100644 index 00000000..85748796 --- /dev/null +++ b/src/utils/MergedLayerValidator.js @@ -0,0 +1,43 @@ +let MergedLayerValidator = (function() { + + function validateAdd(mergedElements) { + + let inputShape; + + if (mergedElements.length > 0) { + inputShape = mergedElements[0].outputShape; + } else { + console.error("Merge Layer missing elements."); + } + + for (let i = 0; i < mergedElements.length; i++) { + + let outputShape = mergedElements[i].outputShape; + + for (let j = 0; j < inputShape.length; j++) { + + if (outputShape[j] !== inputShape[j]) { + return false; + } + + } + + } + + return true; + + } + + function validate(operator, mergedElements) { + if (operator === "add") { + return validateAdd(mergedElements); + } + } + + return { + validate: validate + } + +})(); + +export { MergedLayerValidator }; \ No newline at end of file diff --git a/src/utils/MergedShapeGenerator.js b/src/utils/MergedShapeGenerator.js new file mode 100644 index 00000000..08b86755 --- /dev/null +++ b/src/utils/MergedShapeGenerator.js @@ -0,0 +1,25 @@ +let MergedShapeGenerator = (function() { + + function getAddShape(mergedElements) { + + return mergedElements[0].outputShape; + + } + + function getShape(operator, mergedElements) { + + if (operator === "add") { + return getAddShape(mergedElements); + } + + } + + return { + + getShape: getShape + + } + +})(); + +export { MergedShapeGenerator }; \ No newline at end of file diff --git a/src/utils/TextureProvider.js b/src/utils/TextureProvider.js new file mode 100644 index 00000000..6661e2eb --- /dev/null +++ b/src/utils/TextureProvider.js @@ -0,0 +1,24 @@ +import { CloseData } from "../assets/image/CloseData"; +import { PlusData } from "../assets/image/Plus"; + +let TextureProvider = (function() { + + function getTexture(name) { + + if (name === "close") { + return CloseData; + } else if (name === "add") { + return PlusData; + } + + } + + return { + + getTexture: getTexture + + } + +})(); + +export { TextureProvider }; \ No newline at end of file diff --git a/src/vis-model/Sequential.js b/src/vis-model/Sequential.js index 737f9d43..f6349741 100644 --- a/src/vis-model/Sequential.js +++ b/src/vis-model/Sequential.js @@ -29,9 +29,11 @@ Sequential.prototype = Object.assign(Object.create(AbstractComposite.prototype), if (this.layers.length !== 0) { - let tailLayer = this.layers[this.layers.length - 1]; - layer.setLastLayer(tailLayer); - tailLayer.setNextLayer(layer); + if (!layer.isMerged) { + let tailLayer = this.layers[this.layers.length - 1]; + layer.setLastLayer(tailLayer); + } + } layer.setEnvironment(this.scene); @@ -232,7 +234,8 @@ Sequential.prototype = Object.assign(Object.create(AbstractComposite.prototype), }; let hookHandler = new LineHook(hookPos); - model.scene.add(hookHandler.getElement()); + // 暂时先不把hook加到场景中 + // model.scene.add(hookHandler.getElement()); hookHandlerList.push(hookHandler); diff --git a/test/test.html b/test/test.html index 04521aeb..9124d078 100644 --- a/test/test.html +++ b/test/test.html @@ -45,22 +45,40 @@ shape: [28, 28, 1] })); - model.add(new TSP.layers.Conv2d({ - kernelSize: 2, - filters: 3, - strides: 1, - padding: "same" - })); + let layer1 = new TSP.layers.Conv2d({ + kernelSize: 2, + filters: 3, + strides: 1, + padding: "same" + }); - model.add(new TSP.layers.Activation3d({ - activation: "test" - })); + let layer2 = new TSP.layers.Conv2d({ + kernelSize: 2, + filters: 3, + strides: 1, + padding: "same" + }); + + let addLayer = TSP.Add([layer1, layer2]); + +// console.log(addLayer); +// + model.add(layer1); + model.add(layer2); + + model.add(addLayer); + +// model.add(new TSP.layers.Activation3d({ +// activation: "test" +// })); model.init(function(){ }); +// console.log(TSP.Add(1, 2)); +