Refactoring the ShaderLab Compiler Module using the LALR Algorithm fo…

…r Enhanced GLSL Compatibility and Reduced Build Size (#2113)

* refactor: refactoring the ShaderLab Compiler Module

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Install
         run: pnpm install
-      - run: npm run build
+      - run: npm run build:editor
 
   codecov:
     runs-on: macos-latest
@@ -66,7 +66,7 @@ jobs:
       - name: Install
         run: pnpm install
       - name: Build
-        run: npm run build
+        run: npm run build:editor
       - name: Test
         run: npm run test-cov
       - run: pnpm install codecov -w
@@ -82,38 +82,38 @@ jobs:
         node-version: [16.x]
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        lfs: true
-    - name: Install pnpm
-      uses: pnpm/action-setup@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Run Cypress Tests
-      uses: cypress-io/github-action@v5
-      with:
-        build: npm run build
-        start: npm run e2e:case
-        wait-on: 'http://localhost:5175'
-        wait-on-timeout: 120 
-        browser: chrome
-    - name: Upload Diff
-      if: failure()
-      uses: actions/upload-artifact@v3
-      with:
-        name: cypress-diff
-        path: e2e/diff/
-    - name: Upload Origin
-      if: failure()
-      uses: actions/upload-artifact@v3
-      with:
-        name: cypress-origin
-        path: e2e/fixtures/originImage
-    - name: Upload Screenshots
-      if: failure()
-      uses: actions/upload-artifact@v3
-      with:
-        name: cypress-screenshots
-        path: e2e/downloads/
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Run Cypress Tests
+        uses: cypress-io/github-action@v5
+        with:
+          build: npm run build:editor
+          start: npm run e2e:case
+          wait-on: "http://localhost:5175"
+          wait-on-timeout: 120
+          browser: chrome
+      - name: Upload Diff
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: cypress-diff
+          path: e2e/diff/
+      - name: Upload Origin
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: cypress-origin
+          path: e2e/fixtures/originImage
+      - name: Upload Screenshots
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: cypress-screenshots
+          path: e2e/downloads/

.gitignore

@@ -35,3 +35,7 @@ e2e/.dev/mpa
 .husky/post-commit
 .husky/pre-push
 .husky/post-merge
+
+# For bison generated files used by ShaderLab
+*.tab.c
+*.output

e2e/case/material-shaderLab.ts

@@ -31,11 +31,11 @@ const normalShaderSource = `Shader "Triangle" {
 
       struct a2v {
         vec4 POSITION;
-      }
+      };
 
       struct v2f {
         vec3 v_color;
-      }
+      };
 
       VertexShader = vert;
       FragmentShader = frag;

package.json

@@ -9,14 +9,19 @@
     "test-debug": "cross-env TS_NODE_PROJECT=tsconfig.tests.json floss --path tests -r ts-node/register --debug",
     "test-cov": "cross-env TS_NODE_PROJECT=tsconfig.tests.json IS_COV=1 nyc --reporter=lcov floss --path tests -r ts-node/register",
     "build": "npm run b:module && npm run b:types",
+    "build:editor": "npm run b:module:editor && npm run b:types",
     "lint": "eslint packages/*/src --ext .ts",
-    "watch": "cross-env NODE_ENV=development BUILD_TYPE=MODULE rollup -cw -m inline",
-    "watch:umd": "cross-env NODE_ENV=development BUILD_TYPE=UMD rollup -cw -m inline",
+    "watch": "cross-env NODE_ENV=release BUILD_TYPE=MODULE rollup -cw -m inline",
+    "watch:editor": "cross-env NODE_ENV=editor BUILD_TYPE=MODULE rollup -cw -m inline",
+    "watch:umd": "cross-env NODE_ENV=release BUILD_TYPE=UMD rollup -cw -m inline",
+    "watch:umd:editor": "cross-env NODE_ENV=editor BUILD_TYPE=UMD rollup -cw -m inline",
     "b:types": "pnpm -r --filter=./packages/* run b:types",
-    "b:module": "cross-env BUILD_TYPE=MODULE rollup -c",
-    "b:umd": "cross-env BUILD_TYPE=UMD rollup -c",
+    "b:module": "cross-env BUILD_TYPE=MODULE NODE_ENV=release rollup -c",
+    "b:module:editor": "cross-env BUILD_TYPE=MODULE NODE_ENV=editor rollup -c",
+    "b:umd": "cross-env BUILD_TYPE=UMD NODE_ENV=release rollup -c",
     "b:miniprogram": "cross-env BUILD_TYPE=MINI rollup -c",
-    "b:all": "npm run b:types && cross-env BUILD_TYPE=ALL rollup -c",
+    "b:all": "cross-env NODE_ENV=release npm run b:types && cross-env BUILD_TYPE=ALL NODE_ENV=release rollup -c",
+    "b:all:editor": "cross-env NODE_ENV=editor npm run b:types && cross-env BUILD_TYPE=ALL rollup -c",
     "clean": "pnpm -r exec rm -rf dist && pnpm -r exec rm -rf types",
     "e2e:case": "pnpm -C ./e2e run case",
     "e2e": "cypress run --browser chrome --headless",
@@ -36,32 +41,33 @@
     "@types/chai-spies": "^1.0.3",
     "@types/mocha": "^8.0.0",
     "@types/node": "^18.7.16",
+    "@types/webxr": "latest",
     "@typescript-eslint/eslint-plugin": "^6.1.0",
     "@typescript-eslint/parser": "^6.1.0",
     "chai": "^4.3.6",
     "chai-spies": "^1.0.0",
     "cross-env": "^5.2.0",
+    "cypress": "^12.17.1",
+    "cypress-recurse": "^1.23.0",
     "electron": "^13",
     "eslint": "^8.44.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^5.0.0",
     "floss": "^5.0.1",
+    "fs-extra": "^10.1.0",
+    "husky": "^8.0.0",
     "lint-staged": "^10.5.3",
     "nyc": "^15.1.0",
+    "odiff-bin": "^2.5.0",
     "prettier": "^3.0.0",
     "rollup": "^2.36.1",
     "rollup-plugin-glslify": "^1.2.0",
+    "rollup-plugin-jscc": "^2.0.0",
     "rollup-plugin-modify": "^3.0.0",
     "rollup-plugin-serve": "^1.1.0",
     "rollup-plugin-swc3": "^0.10.1",
     "ts-node": "^10",
-    "typescript": "^5.1.6",
-    "@types/webxr": "latest",
-    "husky": "^8.0.0",
-    "fs-extra": "^10.1.0",
-    "cypress": "^12.17.1",
-    "cypress-recurse": "^1.23.0",
-    "odiff-bin": "^2.5.0"
+    "typescript": "^5.1.6"
   },
   "lint-staged": {
     "*.{ts}": [
@@ -70,4 +76,4 @@
     ]
   },
   "repository": "[email protected]:galacean/runtime.git"
-}
+}

packages/core/src/Engine.ts

@@ -559,7 +559,7 @@ export class Engine extends EventDispatcher {
   protected _initialize(configuration: EngineConfiguration): Promise<Engine> {
     const { shaderLab, physics } = configuration;
 
-    if (shaderLab) {
+    if (shaderLab && !Shader._shaderLab) {
       Shader._shaderLab = shaderLab;
     }
 

packages/core/src/shader/Shader.ts

@@ -89,51 +89,54 @@ export class Shader implements IReferable {
         throw "ShaderLab has not been set up yet.";
       }
 
-      const shaderInfo = Shader._shaderLab.parseShader(nameOrShaderSource);
-      if (shaderMap[shaderInfo.name]) {
-        console.error(`Shader named "${shaderInfo.name}" already exists.`);
+      const shaderContent = Shader._shaderLab._parseShaderContent(nameOrShaderSource);
+      if (shaderMap[shaderContent.name]) {
+        console.error(`Shader named "${shaderContent.name}" already exists.`);
         return;
       }
-      const subShaderList = shaderInfo.subShaders.map((subShaderInfo) => {
-        const passList = subShaderInfo.passes.map((passInfo) => {
-          if (typeof passInfo === "string") {
+      const subShaderList = shaderContent.subShaders.map((subShaderContent) => {
+        const passList = subShaderContent.passes.map((passInfo) => {
+          if (passInfo.isUsePass) {
             // Use pass reference
-            const paths = passInfo.split("/");
+            const paths = passInfo.name.split("/");
             return Shader.find(paths[0])
               ?.subShaders.find((subShader) => subShader.name === paths[1])
               ?.passes.find((pass) => pass.name === paths[2]);
           }
 
-          const shaderPass = new ShaderPass(
+          const shaderPassContent = new ShaderPass(
             passInfo.name,
-            passInfo.vertexSource,
-            passInfo.fragmentSource,
+            passInfo.contents,
+            passInfo.vertexEntry,
+            passInfo.fragmentEntry,
             passInfo.tags
           );
+
           const renderStates = passInfo.renderStates;
           const renderState = new RenderState();
-          shaderPass._renderState = renderState;
 
+          shaderPassContent._renderState = renderState;
           // Parse const render state
-          const constRenderStateInfo = renderStates[0];
-          for (let k in constRenderStateInfo) {
-            Shader._applyConstRenderStates(renderState, <RenderStateElementKey>parseInt(k), constRenderStateInfo[k]);
+          const { constantMap, variableMap } = renderStates;
+          for (let k in constantMap) {
+            Shader._applyConstRenderStates(renderState, <RenderStateElementKey>parseInt(k), constantMap[k]);
           }
 
           // Parse variable render state
-          const variableRenderStateInfo = renderStates[1];
           const renderStateDataMap = {} as Record<number, ShaderProperty>;
-          for (let k in variableRenderStateInfo) {
-            renderStateDataMap[k] = ShaderProperty.getByName(variableRenderStateInfo[k]);
+          for (let k in variableMap) {
+            renderStateDataMap[k] = ShaderProperty.getByName(variableMap[k]);
           }
-          shaderPass._renderStateDataMap = renderStateDataMap;
-          return shaderPass;
+          shaderPassContent._renderStateDataMap = renderStateDataMap;
+
+          return shaderPassContent;
         });
-        return new SubShader(shaderInfo.name, passList, subShaderInfo.tags);
+
+        return new SubShader(subShaderContent.name, passList, subShaderContent.tags);
       });
 
-      shader = new Shader(shaderInfo.name, subShaderList);
-      shaderMap[shaderInfo.name] = shader;
+      shader = new Shader(shaderContent.name, subShaderList);
+      shaderMap[shaderContent.name] = shader;
       return shader;
     } else {
       if (shaderMap[nameOrShaderSource]) {

packages/core/src/shader/ShaderMacro.ts

@@ -5,7 +5,7 @@ import { ShaderMacroCollection } from "./ShaderMacroCollection";
  */
 export class ShaderMacro {
   /** @internal */
-  static _macroMaskMap: string[][] = [];
+  static _macroMaskMap: ShaderMacro[][] = [];
   /** @internal */
   static _macroNameIdMap: Record<string, number> = Object.create(null);
 
@@ -41,9 +41,9 @@ export class ShaderMacro {
       ShaderMacro._macroMap[key] = macro;
       if (index == maskMap.length) {
         maskMap.length++;
-        maskMap[index] = new Array<string>(32);
+        maskMap[index] = new Array<ShaderMacro>(32);
       }
-      maskMap[index][bit] = key;
+      maskMap[index][bit] = macro;
       ShaderMacro._macroCounter++;
     }
     return macro;
@@ -52,7 +52,7 @@ export class ShaderMacro {
   /**
    * @internal
    */
-  static _getNamesByMacros(macros: ShaderMacroCollection, out: string[]): void {
+  static _getMacrosElements(macros: ShaderMacroCollection, out: ShaderMacro[]): void {
     const maskMap = ShaderMacro._macroMaskMap;
     const mask = macros._mask;
     out.length = 0;