Add tfjs wasm backend.

docs/yoha.itfjswasmbackendconfig.md

@@ -0,0 +1,20 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [@handtracking.io/yoha](./yoha.md) &gt; [ITfjsWasmBackendConfig](./yoha.itfjswasmbackendconfig.md)
+
+## ITfjsWasmBackendConfig interface
+
+Configuration that is specific to the tfjs wasm backend.
+
+<b>Signature:</b>
+
+```typescript
+export interface ITfjsWasmBackendConfig 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [wasmPaths](./yoha.itfjswasmbackendconfig.wasmpaths.md) | string | See https://github.com/tensorflow/tfjs/tree/master/tfjs-backend-wasm\#using-bundlers |
+

docs/yoha.itfjswasmbackendconfig.wasmpaths.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [@handtracking.io/yoha](./yoha.md) &gt; [ITfjsWasmBackendConfig](./yoha.itfjswasmbackendconfig.md) &gt; [wasmPaths](./yoha.itfjswasmbackendconfig.wasmpaths.md)
+
+## ITfjsWasmBackendConfig.wasmPaths property
+
+See https://github.com/tensorflow/tfjs/tree/master/tfjs-backend-wasm\#using-bundlers
+
+<b>Signature:</b>
+
+```typescript
+wasmPaths: string;
+```

docs/yoha.iyohatfjsmodelblobs.md

@@ -18,4 +18,5 @@ export interface IYohaTfjsModelBlobs
 |  --- | --- | --- |
 |  [box](./yoha.iyohatfjsmodelblobs.box.md) | [IBlobs](./yoha.iblobs.md) | The file blobs of the box model for detecting initial hand position within stream. |
 |  [lan](./yoha.iyohatfjsmodelblobs.lan.md) | [IBlobs](./yoha.iblobs.md) | The file blobs of the landmark model for detecting landmark locations and detecting hand poses. |
+|  [modelType](./yoha.iyohatfjsmodelblobs.modeltype.md) | 'tfjs' | This field adds some type safety to protect against mismatching blobs/backends. |
 

docs/yoha.iyohatfjsmodelblobs.modeltype.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [@handtracking.io/yoha](./yoha.md) &gt; [IYohaTfjsModelBlobs](./yoha.iyohatfjsmodelblobs.md) &gt; [modelType](./yoha.iyohatfjsmodelblobs.modeltype.md)
+
+## IYohaTfjsModelBlobs.modelType property
+
+This field adds some type safety to protect against mismatching blobs/backends.
+
+<b>Signature:</b>
+
+```typescript
+modelType: 'tfjs';
+```

docs/yoha.md

@@ -15,7 +15,8 @@ Yoha hand tracking library for precision interactions.
 |  [CreateVideoElementFromStream(stream)](./yoha.createvideoelementfromstream.md) | Creates a <code>&lt;video&gt;</code> element for the given stream. |
 |  [DownloadMultipleYohaTfjsModelBlobs(boxUrl, lanUrl, progressCb)](./yoha.downloadmultipleyohatfjsmodelblobs.md) | Downloads the Yoha tfjs models. |
 |  [MirrorCoordinatesHorizontally(coords)](./yoha.mirrorcoordinateshorizontally.md) | Mirrors result coordinates along the y axis i.e. <code>[x, y]</code> becomes <code>[1 - x, y]</code>. |
-|  [StartTfjsWebglEngine(config, trackSource, yohaModels, resCb)](./yoha.starttfjswebglengine.md) | Starts an analysis loop on a track source (e.g. a <code>&lt;video&gt;</code> element) using the TFJS WebGl backend. |
+|  [StartTfjsWasmEngine(engineConfig, backendConfig, trackSource, yohaModels, resCb)](./yoha.starttfjswasmengine.md) | Starts an analysis loop on a track source (e.g. a <code>&lt;video&gt;</code> element) using the tfjs wasm backend. |
+|  [StartTfjsWebglEngine(engineConfig, trackSource, yohaModels, resCb)](./yoha.starttfjswebglengine.md) | Starts an analysis loop on a track source (e.g. a <code>&lt;video&gt;</code> element) using the tfjs webgl backend. |
 
 ## Interfaces
 
@@ -26,6 +27,7 @@ Yoha hand tracking library for precision interactions.
 |  [IMediaStreamResult](./yoha.imediastreamresult.md) | Result of trying to obtain a MediaStream. |
 |  [IPoseProbabilities](./yoha.iposeprobabilities.md) | The detected hand poses. |
 |  [IStopEngineCb](./yoha.istopenginecb.md) | A callback that when invoked stops the engine. |
+|  [ITfjsWasmBackendConfig](./yoha.itfjswasmbackendconfig.md) | Configuration that is specific to the tfjs wasm backend. |
 |  [ITrackResult](./yoha.itrackresult.md) | Hand tracking results. |
 |  [IYohaTfjsModelBlobs](./yoha.iyohatfjsmodelblobs.md) | The two models required for running the Yoha engine. |
 

docs/yoha.starttfjswasmengine.md

@@ -0,0 +1,30 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [@handtracking.io/yoha](./yoha.md) &gt; [StartTfjsWasmEngine](./yoha.starttfjswasmengine.md)
+
+## StartTfjsWasmEngine() function
+
+Starts an analysis loop on a track source (e.g. a `<video>` element) using the tfjs wasm backend.
+
+<b>Signature:</b>
+
+```typescript
+export declare function StartTfjsWasmEngine(engineConfig: IEngineConfig, backendConfig: ITfjsWasmBackendConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  engineConfig | [IEngineConfig](./yoha.iengineconfig.md) | Engine configuration. |
+|  backendConfig | [ITfjsWasmBackendConfig](./yoha.itfjswasmbackendconfig.md) | Backend configuration. |
+|  trackSource | [ITrackSource](./yoha.itracksource.md) | The element to be analyzed. |
+|  yohaModels | [IYohaTfjsModelBlobs](./yoha.iyohatfjsmodelblobs.md) | File blobs of the AI models required for the engine to run. |
+|  resCb | [ITrackResultCb](./yoha.itrackresultcb.md) | Callback that is called with hand tracking results. The callback may be called with high frequency. |
+
+<b>Returns:</b>
+
+Promise&lt;[IStopEngineCb](./yoha.istopenginecb.md)<!-- -->&gt;
+
+Promise that resolves with a callback that can be used to stop the analysis.
+

docs/yoha.starttfjswebglengine.md

@@ -4,19 +4,19 @@
 
 ## StartTfjsWebglEngine() function
 
-Starts an analysis loop on a track source (e.g. a `<video>` element) using the TFJS WebGl backend.
+Starts an analysis loop on a track source (e.g. a `<video>` element) using the tfjs webgl backend.
 
 <b>Signature:</b>
 
 ```typescript
-export declare function StartTfjsWebglEngine(config: IEngineConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
+export declare function StartTfjsWebglEngine(engineConfig: IEngineConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
 ```
 
 ## Parameters
 
 |  Parameter | Type | Description |
 |  --- | --- | --- |
-|  config | [IEngineConfig](./yoha.iengineconfig.md) | Engine configuration. |
+|  engineConfig | [IEngineConfig](./yoha.iengineconfig.md) | Engine configuration. |
 |  trackSource | [ITrackSource](./yoha.itracksource.md) | The element to be analyzed. |
 |  yohaModels | [IYohaTfjsModelBlobs](./yoha.iyohatfjsmodelblobs.md) | File blobs of the AI models required for the engine to run. |
 |  resCb | [ITrackResultCb](./yoha.itrackresultcb.md) | Callback that is called with hand tracking results. The callback may be called with high frequency. |

package.json

@@ -34,6 +34,7 @@
   "devDependencies": {
     "@microsoft/api-documenter": "^7.14.0",
     "@microsoft/api-extractor": "^7.19.3",
+    "@types/emscripten": "^1.39.6",
     "@types/three": "^0.132.1",
     "@typescript-eslint/eslint-plugin": "^5.10.2",
     "@typescript-eslint/parser": "^5.10.2",
@@ -49,6 +50,7 @@
     "webpack-dev-server": "^4.3.0"
   },
   "dependencies": {
+    "@tensorflow/tfjs-backend-wasm": "^3.13.0",
     "@tensorflow/tfjs-backend-webgl": "^3.12.0",
     "@tensorflow/tfjs-converter": "^3.12.0",
     "@tensorflow/tfjs-core": "^3.12.0"

report.api.md

@@ -53,6 +53,11 @@ export interface IStopEngineCb {
     (): void;
 }
 
+// @public
+export interface ITfjsWasmBackendConfig {
+    wasmPaths: string;
+}
+
 // @public
 export interface ITrackResult {
     coordinates: number[][];
@@ -71,6 +76,7 @@ export type ITrackSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageEleme
 export interface IYohaTfjsModelBlobs {
     box: IBlobs;
     lan: IBlobs;
+    modelType: 'tfjs';
 }
 
 // @public
@@ -91,6 +97,9 @@ export function MirrorCoordinatesHorizontally(coords: number[][]): number[][];
 export type ObjValues<T> = T[keyof T];
 
 // @public
-export function StartTfjsWebglEngine(config: IEngineConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
+export function StartTfjsWasmEngine(engineConfig: IEngineConfig, backendConfig: ITfjsWasmBackendConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
+
+// @public
+export function StartTfjsWebglEngine(engineConfig: IEngineConfig, trackSource: ITrackSource, yohaModels: IYohaTfjsModelBlobs, resCb: ITrackResultCb): Promise<IStopEngineCb>;
 
 ```

src/core/engine/base.ts

@@ -13,9 +13,9 @@ import {
 } from '../pre_model/preproc_comp';
 import {IPreprocessCb} from '../pre_model/preproc';
 import {IModelCb} from '../model/base';
-
 import {ITrackSource} from '../track_source';
 import {RequestAnimationFrame} from '../../util/animation_frame';
+import {ApplyConfigDefaults} from '../../util/config_helper';
 
 /**
  * @public
@@ -107,10 +107,7 @@ export async function StartEngine(
   lanCb: IModelCb,
   resCb: ITrackResultCb
 ): Promise<IStopEngineCb> {
-  if (!config) {
-    config = JSON.parse(JSON.stringify(DEFAULT_CONFIG));
-  }
-  ApplyConfigDefaults(DEFAULT_CONFIG,  config);
+  config = ApplyConfigDefaults(DEFAULT_CONFIG,  config);
   const aspectRatio = [ trackSource.width, trackSource.height ];
 
   let stopped = false;
@@ -177,24 +174,3 @@ export function ApplyPostProcessingToCoordinates(
   return coords;
 }
 
-function ApplyConfigDefaults(
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  srcConfig: any,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  trgConfig: any
-): void {
-  if (srcConfig === null || srcConfig === undefined || 
-      trgConfig === null || trgConfig === undefined) {
-    return;
-  }
-  if (typeof srcConfig !== 'object' || typeof trgConfig !== 'object') {
-    return;
-  }
-  for (const key of Object.keys(srcConfig)) {
-    if (key in trgConfig) {
-      ApplyConfigDefaults(srcConfig[key], trgConfig[key]);
-    } else if (key) {
-      trgConfig[key] = srcConfig[key];
-    }
-  }
-}

src/core/engine/tfjs_webgl_engine.ts → src/core/engine/tfjs_base.ts

@@ -8,11 +8,11 @@ import {
 import {ITrackSource} from '../track_source';
 
 import {
-  TfjsBackendType,
-  IYohaTfjsModelBlobs,
-  CreateTfjsModelFromModelBlobs,
+  IInternalTfjsBackendConfig,
   CreateModelCbFromTfjsModel,
   GetInputDimensionsFromTfjsModel,
+  CreateTfjsModelFromModelBlobs,
+  IYohaTfjsModelBlobs,
 } from '../model/tfjs';
 
 import {
@@ -21,26 +21,28 @@ import {
 
 /**
  * @public
- * Starts an analysis loop on a track source (e.g. a `<video>` element) using the TFJS WebGl 
+ * Starts an analysis loop on a track source (e.g. a `<video>` element) using a tfjs 
  * backend.
  *
- * @param config - Engine configuration.
+ * @param engineConfig - Engine configuration.
+ * @param backendConfig - Backend configuration.
  * @param trackSource - The element to be analyzed.
  * @param resCb - Callback that is called with hand tracking results. The callback may be called
  *                with high frequency.
  * @param yohaModels - File blobs of the AI models required for the engine to run.
  *
  * @returns Promise that resolves with a callback that can be used to stop the analysis.
  */
-export async function StartTfjsWebglEngine(
-  config: IEngineConfig, 
-  trackSource: ITrackSource, 
+export async function StartTfjsEngine(
+  engineConfig: IEngineConfig, 
+  backendConfig: IInternalTfjsBackendConfig,
   yohaModels: IYohaTfjsModelBlobs,
+  trackSource: ITrackSource, 
   resCb: ITrackResultCb,
 ) : Promise<IStopEngineCb> {
   const [boxModel, lanModel] = await Promise.all([
-    CreateTfjsModelFromModelBlobs(yohaModels.box, TfjsBackendType.WEBGL),  
-    CreateTfjsModelFromModelBlobs(yohaModels.lan, TfjsBackendType.WEBGL),
+    CreateTfjsModelFromModelBlobs(yohaModels.box, backendConfig),  
+    CreateTfjsModelFromModelBlobs(yohaModels.lan, backendConfig),
   ]);
 
   const boxDims = GetInputDimensionsFromTfjsModel(boxModel);
@@ -59,5 +61,5 @@ export async function StartTfjsWebglEngine(
   const boxCb = CreateModelCbFromTfjsModel(boxModel, true);
   const lanCb = CreateModelCbFromTfjsModel(lanModel, true);
 
-  return StartEngine(config, trackSource, preprocCb, boxCb, lanCb, resCb);
+  return StartEngine(engineConfig, trackSource, preprocCb, boxCb, lanCb, resCb);
 }

src/core/engine/tfjs_wasm.ts

@@ -0,0 +1,45 @@
+import {
+  IEngineConfig,
+  ITrackResultCb,
+  IStopEngineCb,
+} from './base';
+
+import {ITrackSource} from '../track_source';
+
+import {
+  IYohaTfjsModelBlobs,
+  ITfjsWasmBackendConfig,
+  TfjsBackendType,
+} from '../model/tfjs';
+
+import {StartTfjsEngine} from './tfjs_base';
+
+/**
+ * @public
+ * Starts an analysis loop on a track source (e.g. a `<video>` element) using the tfjs wasm 
+ * backend.
+ *
+ * @param engineConfig - Engine configuration.
+ * @param backendConfig - Backend configuration.
+ * @param trackSource - The element to be analyzed.
+ * @param resCb - Callback that is called with hand tracking results. The callback may be called
+ *                with high frequency.
+ * @param yohaModels - File blobs of the AI models required for the engine to run.
+ *
+ * @returns Promise that resolves with a callback that can be used to stop the analysis.
+ */
+export async function StartTfjsWasmEngine(
+  engineConfig: IEngineConfig, 
+  backendConfig: ITfjsWasmBackendConfig,
+  trackSource: ITrackSource, 
+  yohaModels: IYohaTfjsModelBlobs,
+  resCb: ITrackResultCb,
+) : Promise<IStopEngineCb> {
+  return StartTfjsEngine(
+    engineConfig, 
+    {...backendConfig, backendType: TfjsBackendType.WASM}, 
+    yohaModels, 
+    trackSource, 
+    resCb
+  );
+}

src/core/engine/tfjs_webgl.ts

@@ -0,0 +1,42 @@
+import {
+  IEngineConfig,
+  ITrackResultCb,
+  IStopEngineCb,
+} from './base';
+
+import {ITrackSource} from '../track_source';
+
+import {
+  TfjsBackendType,
+  IYohaTfjsModelBlobs,
+} from '../model/tfjs';
+
+import {StartTfjsEngine} from './tfjs_base';
+
+/**
+ * @public
+ * Starts an analysis loop on a track source (e.g. a `<video>` element) using the tfjs webgl 
+ * backend.
+ *
+ * @param engineConfig - Engine configuration.
+ * @param trackSource - The element to be analyzed.
+ * @param resCb - Callback that is called with hand tracking results. The callback may be called
+ *                with high frequency.
+ * @param yohaModels - File blobs of the AI models required for the engine to run.
+ *
+ * @returns Promise that resolves with a callback that can be used to stop the analysis.
+ */
+export async function StartTfjsWebglEngine(
+  engineConfig: IEngineConfig, 
+  trackSource: ITrackSource, 
+  yohaModels: IYohaTfjsModelBlobs,
+  resCb: ITrackResultCb,
+) : Promise<IStopEngineCb> {
+  return StartTfjsEngine(
+    engineConfig, 
+    {backendType: TfjsBackendType.WEBGL}, 
+    yohaModels, 
+    trackSource, 
+    resCb
+  );
+}

src/core/model/base.ts

@@ -157,6 +157,7 @@ export interface IBlobs {
  *                 by {@link downloadModelsCb}.
  * @param progressCb - A callback that is called with the cumulative download progress for all
  *                     models.
+ * @param downloadModelsCb - A callback that is called to download multiple models simultaneously.
  */
 export async function DownloadMultipleYohaModelBlobs(
   boxUrl: string,