From cd424cf3285118e9d73824e2245d6facea1327ce Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 00:58:43 +0200 Subject: [PATCH 01/12] Extended window provider with minimize and restore methods --- .../lib/window-provider.interface.ts | 10 ++++++++++ providers/libnut/lib/libnut-window.class.ts | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/core/provider-interfaces/lib/window-provider.interface.ts b/core/provider-interfaces/lib/window-provider.interface.ts index 288a59a..45a2e01 100644 --- a/core/provider-interfaces/lib/window-provider.interface.ts +++ b/core/provider-interfaces/lib/window-provider.interface.ts @@ -49,4 +49,14 @@ export interface WindowProviderInterface { * {@link resizeWindow} Resizes the window addressed via its window handle to a new {@link Size} */ resizeWindow(windowHandle: number, newSize: Size): Promise; + + /** + * {@link minimizeWindow} Minimizes the window addressed via its window handle + */ + minimizeWindow(windowHandle: number): Promise; + + /** + * {@link restoreWindow} Restores a window addressed via its window handle + */ + restoreWindow(windowHandle: number): Promise; } diff --git a/providers/libnut/lib/libnut-window.class.ts b/providers/libnut/lib/libnut-window.class.ts index bb24712..1a3ae35 100644 --- a/providers/libnut/lib/libnut-window.class.ts +++ b/providers/libnut/lib/libnut-window.class.ts @@ -80,4 +80,12 @@ export default class WindowAction implements WindowProviderInterface { } }); } + + minimizeWindow(_: number): Promise { + throw new Error("Method not implemented in libnut."); + } + + restoreWindow(_: number): Promise { + throw new Error("Method not implemented in libnut."); + } } From 46aed4bca3c9077d53f62ffeaf413f062d7a6488 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:00:40 +0200 Subject: [PATCH 02/12] Moved window interface to types folder, extended it with element query definitions --- core/shared/lib/objects/window.interface.ts | 15 --------- core/shared/lib/types/window.interface.ts | 35 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) delete mode 100644 core/shared/lib/objects/window.interface.ts create mode 100644 core/shared/lib/types/window.interface.ts diff --git a/core/shared/lib/objects/window.interface.ts b/core/shared/lib/objects/window.interface.ts deleted file mode 100644 index fd6160e..0000000 --- a/core/shared/lib/objects/window.interface.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Region } from "./region.class"; -import { Point } from "./point.class"; -import { Size } from "./size.class"; - -export interface WindowInterface { - getTitle(): Promise; - - getRegion(): Promise; - - move(newOrigin: Point): Promise; - - resize(newSize: Size): Promise; - - focus(): Promise; -} \ No newline at end of file diff --git a/core/shared/lib/types/window.interface.ts b/core/shared/lib/types/window.interface.ts new file mode 100644 index 0000000..2e56c4e --- /dev/null +++ b/core/shared/lib/types/window.interface.ts @@ -0,0 +1,35 @@ +import { Point, Region, Size } from "../objects"; +import { WindowElement } from "./window-element.interface"; +import { PointResultFindInput, RegionResultFindInput, WindowElementResultFindInput } from "./index"; + +export interface WindowInterface { + getTitle(): Promise; + + getRegion(): Promise; + + move(newOrigin: Point): Promise; + + resize(newSize: Size): Promise; + + focus(): Promise; + + getElements(maxElements?: number): Promise; + + find( + searchInput: WindowElementResultFindInput | Promise + ): Promise; + + findAll( + searchInput: WindowElementResultFindInput | Promise + ): Promise; +} + +export type WindowedFindInput = + | RegionResultFindInput + | WindowElementResultFindInput + | PointResultFindInput; +export type WindowedFindResult = Region | Point | WindowElement; + +export type WindowElementCallback = ( + target: WindowElement +) => void | Promise; From 18f2d087cdfc364cdb1727e00a432e6c53e6ee18 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:01:10 +0200 Subject: [PATCH 03/12] Defined intefaces for WindowElement objects and their description --- .../lib/types/window-element.interface.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 core/shared/lib/types/window-element.interface.ts diff --git a/core/shared/lib/types/window-element.interface.ts b/core/shared/lib/types/window-element.interface.ts new file mode 100644 index 0000000..8a7af31 --- /dev/null +++ b/core/shared/lib/types/window-element.interface.ts @@ -0,0 +1,23 @@ +import { Region } from "../objects"; + +export interface WindowElement { + type?: string; + region?: Region; + title?: string; + value?: string; + isFocused?: boolean; + selectedText?: string; + isEnabled?: boolean; + role?: string; + subRole?: string; + children?: WindowElement[]; +} + +export interface WindowElementDescription { + id?: string; + role?: string; + type?: string; + title?: string | RegExp; + value?: string | RegExp; + selectedText?: string | RegExp; +} \ No newline at end of file From 7121e9c836eedeaf90018114803a99a9cb45020b Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:01:35 +0200 Subject: [PATCH 04/12] Introduces window-element query type --- core/shared/lib/objects/query.class.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/core/shared/lib/objects/query.class.ts b/core/shared/lib/objects/query.class.ts index 57b4442..0b68526 100644 --- a/core/shared/lib/objects/query.class.ts +++ b/core/shared/lib/objects/query.class.ts @@ -1,4 +1,5 @@ import { RGBA } from "./rgba.class"; +import { WindowElementDescription } from "../types"; type Query = | { @@ -27,6 +28,12 @@ type Query = type: "color"; by: { color: RGBA; + } +} | { + id: string; + type: "window-element"; + by: { + description: WindowElementDescription; }; }; @@ -50,6 +57,12 @@ export type LineQuery = Extract; */ export type WindowQuery = Extract; +/** + * A window element query is a query that searches for an element of a window. + * It will be processed by an {@link ElementInspectionProviderInterface} instance. + */ +export type WindowElementQuery = Extract; + /** * A color query is a query that searches for a certain RGBA color on screen. * It will be processed by an {@link ColorFinderInterface} instance. @@ -99,3 +112,13 @@ export const isWindowQuery = ( ): possibleQuery is WindowQuery => { return possibleQuery?.type === "window"; }; + +/** + * Type guard for {@link WindowElementQuery} + * @param possibleQuery A possible window element query + */ +export const isWindowElementQuery = ( + possibleQuery: any +): possibleQuery is WindowElementQuery => { + return possibleQuery?.type === "window-element"; +}; From a34f4f776158b7429059e156f4c81e6837322ca4 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:02:10 +0200 Subject: [PATCH 05/12] Introduced ElementInspectionProviderInterface --- .../lib/element-inspection-provider.interface.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 core/provider-interfaces/lib/element-inspection-provider.interface.ts diff --git a/core/provider-interfaces/lib/element-inspection-provider.interface.ts b/core/provider-interfaces/lib/element-inspection-provider.interface.ts new file mode 100644 index 0000000..8e93ed2 --- /dev/null +++ b/core/provider-interfaces/lib/element-inspection-provider.interface.ts @@ -0,0 +1,13 @@ +import { WindowElement, WindowElementDescription } from "@nut-tree/shared"; + +/** + * An ElementInspectionProvider provides methods to list and inspect window elements + */ +export interface ElementInspectionProviderInterface { + getElements(windowHandle: number, maxElements?: number): Promise; + + findElement(windowHandle: number, description: WindowElementDescription): Promise; + + findElements(windowHandle: number, description: WindowElementDescription): Promise; +} + From de4d616cb84046f0f410dea2a9a98b9089f4cae0 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:02:52 +0200 Subject: [PATCH 06/12] Added ElementInspectionProviderInterface to provider registry --- .../lib/provider/provider-registry.class.ts | 18 +++++++++++++++++- .../lib/provider-registry.interface.ts | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/core/nut.js/lib/provider/provider-registry.class.ts b/core/nut.js/lib/provider/provider-registry.class.ts index 12f50b5..4e44b34 100644 --- a/core/nut.js/lib/provider/provider-registry.class.ts +++ b/core/nut.js/lib/provider/provider-registry.class.ts @@ -12,7 +12,8 @@ import { ScreenProviderInterface, TextFinderInterface, WindowFinderInterface, - WindowProviderInterface + WindowProviderInterface, + ElementInspectionProviderInterface } from "@nut-tree/provider-interfaces"; import ImageReaderImpl from "./io/jimp-image-reader.class"; @@ -45,6 +46,7 @@ class DefaultProviderRegistry implements ProviderRegistry { private _textFinder?: TextFinderInterface; private _windowFinder?: WindowFinderInterface; private _colorFinder?: ColorFinderInterface; + private _windowElementInspector?: ElementInspectionProviderInterface; hasClipboard(): boolean { return this._clipboard != null; @@ -190,6 +192,20 @@ class DefaultProviderRegistry implements ProviderRegistry { this.getLogProvider().trace("Registered new WindowFinder provider", value); }; + getWindowElementInspector = (): ElementInspectionProviderInterface => { + if (this._windowElementInspector) { + return this._windowElementInspector; + } + const error = new Error(`No WindowElementInspector registered`); + this.getLogProvider().error(error); + throw error; + }; + + registerWindowElementInspector = (value: ElementInspectionProviderInterface) => { + this._windowElementInspector = value; + this.getLogProvider().trace("Registered new WindowElementInspector provider", value); + }; + hasImageReader(): boolean { return this._imageReader != null; } diff --git a/core/provider-interfaces/lib/provider-registry.interface.ts b/core/provider-interfaces/lib/provider-registry.interface.ts index 0a88571..6b21f07 100644 --- a/core/provider-interfaces/lib/provider-registry.interface.ts +++ b/core/provider-interfaces/lib/provider-registry.interface.ts @@ -1,6 +1,7 @@ import { ClipboardProviderInterface, ColorFinderInterface, + ElementInspectionProviderInterface, ImageFinderInterface, ImageProcessor, ImageReader, @@ -87,6 +88,10 @@ export interface ProviderRegistry { registerWindowFinder(value: WindowFinderInterface): void; + getWindowElementInspector(): ElementInspectionProviderInterface; + + registerWindowElementInspector(value: ElementInspectionProviderInterface): void; + hasColorFinder(): boolean; getColorFinder(): ColorFinderInterface; From 728baffcec26d9766688d33b4539f8f4615fbb46 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:03:50 +0200 Subject: [PATCH 07/12] Restructured shared code to improve exports --- core/nut.js/lib/screen.class.ts | 17 +++++++---------- core/provider-interfaces/lib/index.ts | 3 ++- core/shared/lib/enums/index.ts | 4 ++++ core/shared/lib/functions/index.ts | 1 + core/shared/lib/index.ts | 19 +++---------------- core/shared/lib/objects/index.ts | 10 ++++++++++ core/shared/lib/types/index.ts | 17 +++-------------- core/shared/lib/types/screen.types.ts | 21 +++++++++++++++++++++ 8 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 core/shared/lib/enums/index.ts create mode 100644 core/shared/lib/functions/index.ts create mode 100644 core/shared/lib/objects/index.ts create mode 100644 core/shared/lib/types/screen.types.ts diff --git a/core/nut.js/lib/screen.class.ts b/core/nut.js/lib/screen.class.ts index 64dc65e..6b3c71d 100644 --- a/core/nut.js/lib/screen.class.ts +++ b/core/nut.js/lib/screen.class.ts @@ -2,6 +2,7 @@ import { cwd } from "process"; import { ColorQuery, FileType, + FindHookCallback, FindInput, FindResult, Image, @@ -13,12 +14,13 @@ import { isWindowQuery, LineQuery, MatchRequest, - MatchResult, + MatchResultCallback, OptionalSearchParameters, Point, PointResultFindInput, Region, RegionResultFindInput, + WindowCallback, WindowResultFindInput, WordQuery } from "@nut-tree/shared"; @@ -34,15 +36,6 @@ import { isRegionResultFindInput } from "./screen-helpers.function"; -export type WindowCallback = (target: Window) => void | Promise; -export type MatchResultCallback = ( - target: MatchResult -) => void | Promise; -export type FindHookCallback = - | WindowCallback - | MatchResultCallback - | MatchResultCallback; - function validateSearchRegion( search: Region, screen: Region, @@ -333,6 +326,10 @@ export class ScreenClass { searchInput: WindowResultFindInput | Promise, params?: OptionalSearchParameters ): Promise; + public async findAll( + searchInput: FindInput | Promise, + params?: OptionalSearchParameters + ): Promise; public async findAll( searchInput: FindInput | Promise, params?: OptionalSearchParameters diff --git a/core/provider-interfaces/lib/index.ts b/core/provider-interfaces/lib/index.ts index d3e1798..461c280 100644 --- a/core/provider-interfaces/lib/index.ts +++ b/core/provider-interfaces/lib/index.ts @@ -13,4 +13,5 @@ export * from './screen-provider.interface'; export * from './text-finder.interface'; export * from './window-finder.interface'; export * from './window-provider.interface'; -export * from './provider-registry.interface'; \ No newline at end of file +export * from './provider-registry.interface'; +export * from './element-inspection-provider.interface'; \ No newline at end of file diff --git a/core/shared/lib/enums/index.ts b/core/shared/lib/enums/index.ts new file mode 100644 index 0000000..a6a2bbd --- /dev/null +++ b/core/shared/lib/enums/index.ts @@ -0,0 +1,4 @@ +export * from "./button.enum"; +export * from "./colormode.enum"; +export * from "./file-type.enum"; +export * from "./key.enum"; diff --git a/core/shared/lib/functions/index.ts b/core/shared/lib/functions/index.ts new file mode 100644 index 0000000..acb0f1e --- /dev/null +++ b/core/shared/lib/functions/index.ts @@ -0,0 +1 @@ +export * from "./imageToJimp.function"; diff --git a/core/shared/lib/index.ts b/core/shared/lib/index.ts index 7c4f7b5..9620882 100644 --- a/core/shared/lib/index.ts +++ b/core/shared/lib/index.ts @@ -1,17 +1,4 @@ -export * from "./enums/button.enum"; -export * from "./enums/colormode.enum"; -export * from "./enums/file-type.enum"; -export * from "./enums/key.enum"; -export * from "./objects/image.class"; -export * from "./objects/match-request.class"; -export * from "./objects/match-result.class"; -export * from "./objects/optionalsearchparameters.class"; -export * from "./objects/point.class"; -export * from "./objects/query.class"; -export * from "./objects/region.class"; -export * from "./objects/rgba.class"; -export * from "./objects/scaled-match-result.class"; -export * from "./objects/size.class"; -export * from "./objects/window.interface"; +export * from "./enums"; +export * from "./objects"; export * from "./types"; -export * from "./functions/imageToJimp.function"; \ No newline at end of file +export * from "./functions"; \ No newline at end of file diff --git a/core/shared/lib/objects/index.ts b/core/shared/lib/objects/index.ts new file mode 100644 index 0000000..19622da --- /dev/null +++ b/core/shared/lib/objects/index.ts @@ -0,0 +1,10 @@ +export * from "./image.class"; +export * from "./match-request.class"; +export * from "./match-result.class"; +export * from "./optionalsearchparameters.class"; +export * from "./point.class"; +export * from "./query.class"; +export * from "./region.class"; +export * from "./rgba.class"; +export * from "./scaled-match-result.class"; +export * from "./size.class"; diff --git a/core/shared/lib/types/index.ts b/core/shared/lib/types/index.ts index 7d62a9e..6ec7bea 100644 --- a/core/shared/lib/types/index.ts +++ b/core/shared/lib/types/index.ts @@ -1,14 +1,3 @@ -import { Image } from "../objects/image.class"; -import { ColorQuery, TextQuery, WindowQuery } from "../objects/query.class"; -import { Region } from "../objects/region.class"; -import { Point } from "../objects/point.class"; -import { WindowInterface } from "../objects/window.interface"; - -export type RegionResultFindInput = Image | TextQuery; -export type PointResultFindInput = ColorQuery; -export type WindowResultFindInput = WindowQuery; -export type FindInput = - | RegionResultFindInput - | WindowResultFindInput - | PointResultFindInput; -export type FindResult = Region | Point | WindowInterface; +export * from "./window-element.interface"; +export * from "./window.interface"; +export * from "./screen.types"; \ No newline at end of file diff --git a/core/shared/lib/types/screen.types.ts b/core/shared/lib/types/screen.types.ts new file mode 100644 index 0000000..865a6c5 --- /dev/null +++ b/core/shared/lib/types/screen.types.ts @@ -0,0 +1,21 @@ +import { ColorQuery, Image, MatchResult, Point, Region, TextQuery, WindowElementQuery, WindowQuery } from "../objects"; +import { WindowInterface } from "./window.interface"; + +export type RegionResultFindInput = Image | TextQuery; +export type PointResultFindInput = ColorQuery; +export type WindowResultFindInput = WindowQuery; +export type WindowElementResultFindInput = WindowElementQuery; +export type FindInput = + | RegionResultFindInput + | WindowResultFindInput + | PointResultFindInput; +export type FindResult = Region | Point | WindowInterface; + +export type WindowCallback = (target: WindowInterface) => void | Promise; +export type MatchResultCallback = ( + target: MatchResult +) => void | Promise; +export type FindHookCallback = + | WindowCallback + | MatchResultCallback + | MatchResultCallback; \ No newline at end of file From f47b91a2bf7fa0b1bd96425d3c7c89dd1cf479ab Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:04:12 +0200 Subject: [PATCH 08/12] Introduced windowElementDescribedBy query --- core/nut.js/index.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/core/nut.js/index.ts b/core/nut.js/index.ts index 91db916..6c995f3 100644 --- a/core/nut.js/index.ts +++ b/core/nut.js/index.ts @@ -8,7 +8,15 @@ import { LineHelper } from "./lib/util/linehelper.class"; import { createWindowApi } from "./lib/window.function"; import providerRegistry from "./lib/provider/provider-registry.class"; import { loadImageResource } from "./lib/imageResources.function"; -import { ColorQuery, LineQuery, RGBA, WindowQuery, WordQuery } from "@nut-tree/shared"; +import { + ColorQuery, + LineQuery, + RGBA, + WindowElementDescription, + WindowElementQuery, + WindowQuery, + WordQuery +} from "@nut-tree/shared"; export { AssertClass, @@ -91,6 +99,16 @@ const windowWithTitle = (title: string | RegExp): WindowQuery => { }; }; +const windowElementDescribedBy = (description: WindowElementDescription): WindowElementQuery => { + return { + type: "window-element", + id: `window-element-described-by-${JSON.stringify(description)}`, + by: { + description + } + }; +}; + const pixelWithColor = (color: RGBA): ColorQuery => { return { type: "color", @@ -122,5 +140,6 @@ export { singleWord, textLine, windowWithTitle, + windowElementDescribedBy, pixelWithColor }; From 16901718899d7036d548e67b3046b2ce3e6656e6 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:33:24 +0200 Subject: [PATCH 09/12] Implemented find, findAll, waitFor and hooks for windows --- core/nut.js/lib/window.class.spec.ts | 174 +++++++++++++++++++++- core/nut.js/lib/window.class.ts | 165 +++++++++++++++++++- core/shared/lib/types/window.interface.ts | 11 +- 3 files changed, 346 insertions(+), 4 deletions(-) diff --git a/core/nut.js/lib/window.class.spec.ts b/core/nut.js/lib/window.class.spec.ts index 45e6bb6..4a4056f 100644 --- a/core/nut.js/lib/window.class.spec.ts +++ b/core/nut.js/lib/window.class.spec.ts @@ -1,7 +1,15 @@ import { Window } from "./window.class"; -import { ProviderRegistry, ScreenProviderInterface, WindowProviderInterface } from "@nut-tree/provider-interfaces"; +import { + ElementInspectionProviderInterface, + LogProviderInterface, + ProviderRegistry, + ScreenProviderInterface, + WindowProviderInterface +} from "@nut-tree/provider-interfaces"; import { mockPartial } from "sneer"; -import { Region } from "@nut-tree/shared"; +import { Region, WindowElement, WindowElementDescription } from "@nut-tree/shared"; +import { windowElementDescribedBy } from "../index"; +import { NoopLogProvider } from "./provider/log/noop-log-provider.class"; describe("Window class", () => { it("should retrieve the window region via provider", async () => { @@ -54,4 +62,166 @@ describe("Window class", () => { expect(windowMock).toHaveBeenCalledTimes(1); expect(windowMock).toHaveBeenCalledWith(mockWindowHandle); }); + + describe("element-inspection", () => { + it("should retrieve the window elements via provider", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + getElements: elementInspectorMock + }); + } + }); + const mockWindowHandle = 123; + const maxElements = 1000; + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + await SUT.getElements(maxElements); + + // THEN + expect(elementInspectorMock).toHaveBeenCalledTimes(1); + expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, maxElements); + }); + + it("should search for window elements via provider", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock, + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description: WindowElementDescription = { + type: "test" + }; + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + await SUT.find(windowElementDescribedBy(description)); + + // THEN + expect(elementInspectorMock).toHaveBeenCalledTimes(1); + expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, description); + }); + + it("should search for multiple elements via provider", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock, + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description: WindowElementDescription = { + type: "test" + }; + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + await SUT.findAll(windowElementDescribedBy(description)); + + // THEN + expect(elementInspectorMock).toHaveBeenCalledTimes(1); + expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, description); + }); + + describe("hooks", () => { + it("should trigger registered hooks", async () => { + // GIVEN + const windowElementType = { type: "testElement" }; + const windowElement = mockPartial(windowElementType); + const hookMock = jest.fn(); + const elementInspectorMock = jest.fn(() => Promise.resolve(windowElement)); + const secondHookMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description: WindowElementDescription = { + type: "test" + }; + const query = windowElementDescribedBy(description); + const SUT = new Window(providerRegistryMock, mockWindowHandle); + SUT.on(query, hookMock); + SUT.on(query, secondHookMock); + + // WHEN + await SUT.find(query); + + // THEN + expect(elementInspectorMock).toHaveBeenCalledTimes(1); + expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, description); + expect(hookMock).toHaveBeenCalledTimes(1); + expect(hookMock).toHaveBeenCalledWith(windowElement); + expect(secondHookMock).toHaveBeenCalledTimes(1); + expect(secondHookMock).toHaveBeenCalledWith(windowElement); + }); + + it("should trigger registered hooks for all matches", async () => { + // GIVEN + const windowElementType = { type: "testElement" }; + const secondElementType = { type: "secondElement" }; + const windowElement = mockPartial(windowElementType); + const secondElement = mockPartial(secondElementType); + const mockMatches = [windowElement, secondElement]; + const hookMock = jest.fn(); + const elementInspectorMock = jest.fn(() => Promise.resolve(mockMatches)); + const secondHookMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description: WindowElementDescription = { + type: "test" + }; + const query = windowElementDescribedBy(description); + const SUT = new Window(providerRegistryMock, mockWindowHandle); + SUT.on(query, hookMock); + SUT.on(query, secondHookMock); + + // WHEN + await SUT.findAll(query); + + // THEN + expect(elementInspectorMock).toHaveBeenCalledTimes(1); + expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, description); + expect(hookMock).toHaveBeenCalledTimes(mockMatches.length); + expect(hookMock).toHaveBeenCalledWith(windowElement); + expect(hookMock).toHaveBeenCalledWith(secondElement); + expect(secondHookMock).toHaveBeenCalledTimes(mockMatches.length); + expect(secondHookMock).toHaveBeenCalledWith(secondElement); + }); + }); + }); }); diff --git a/core/nut.js/lib/window.class.ts b/core/nut.js/lib/window.class.ts index 66ca7ba..24bc3cf 100644 --- a/core/nut.js/lib/window.class.ts +++ b/core/nut.js/lib/window.class.ts @@ -1,11 +1,31 @@ -import { Point, Region, Size, WindowInterface } from "@nut-tree/shared"; +import { + FindHookCallback, + isWindowElementQuery, + OptionalSearchParameters, + Point, + Region, + Size, + WindowedFindInput, + WindowElement, + WindowElementCallback, + WindowElementQuery, + WindowElementResultFindInput, + WindowInterface +} from "@nut-tree/shared"; import { ProviderRegistry } from "@nut-tree/provider-interfaces"; +import { timeout } from "./util/timeout.function"; export class Window implements WindowInterface { + private findHooks: Map; + constructor( private providerRegistry: ProviderRegistry, private windowHandle: number ) { + this.findHooks = new Map< + WindowElementQuery, + WindowElementCallback[] + >(); } get title(): Promise { @@ -65,4 +85,147 @@ export class Window implements WindowInterface { async focus() { return this.providerRegistry.getWindow().focusWindow(this.windowHandle); } + + async getElements(maxElements?: number): Promise { + return this.providerRegistry.getWindowElementInspector().getElements(this.windowHandle, maxElements); + } + + /** + * {@link find} will search for a single occurrence of a given search input in the current window. + * @param searchInput A {@link WindowedFindInput} instance + */ + public async find( + searchInput: WindowElementResultFindInput | Promise + ): Promise { + const needle = await searchInput; + this.providerRegistry.getLogProvider().info(`Searching for ${needle} in window ${this.windowHandle}`); + + try { + if (isWindowElementQuery(needle)) { + this.providerRegistry.getLogProvider().debug(`Running a window element search`); + const windowElement = await this.providerRegistry + .getWindowElementInspector() + .findElement(this.windowHandle, needle.by.description); + const possibleHooks = this.getHooksForInput(needle) || []; + this.providerRegistry + .getLogProvider() + .debug(`${possibleHooks.length} hooks triggered for match`); + for (const hook of possibleHooks) { + this.providerRegistry.getLogProvider().debug(`Executing hook`); + await hook(windowElement); + } + return windowElement; + } + throw new Error( + `Search input is not supported. Please use a valid search input type.` + ); + } catch (e) { + const error = new Error( + `Searching for ${needle.id} failed. Reason: '${e}'` + ); + this.providerRegistry.getLogProvider().error(error); + throw error; + } + } + + /** + * {@link findAll} will search for multiple occurrence of a given search input in the current window. + * @param searchInput A {@link WindowedFindInput} instance + */ + public async findAll( + searchInput: WindowElementResultFindInput | Promise + ): Promise { + // return this.providerRegistry.getWindowElementInspector().findElement(this.windowHandle, description); + const needle = await searchInput; + this.providerRegistry.getLogProvider().info(`Searching for ${needle} in window ${this.windowHandle}`); + + try { + if (isWindowElementQuery(needle)) { + this.providerRegistry.getLogProvider().debug(`Running a window element search`); + const windowElements = await this.providerRegistry + .getWindowElementInspector() + .findElements(this.windowHandle, needle.by.description); + const possibleHooks = this.getHooksForInput(needle) || []; + this.providerRegistry + .getLogProvider() + .debug(`${possibleHooks.length} hooks triggered for match`); + for (const hook of possibleHooks) { + for (const windowElement of windowElements) { + this.providerRegistry.getLogProvider().debug(`Executing hook`); + await hook(windowElement); + } + } + return windowElements; + } + throw new Error( + `Search input is not supported. Please use a valid search input type.` + ); + } catch (e) { + const error = new Error( + `Searching for ${needle.id} failed. Reason: '${e}'` + ); + this.providerRegistry.getLogProvider().error(error); + throw error; + } + } + + /** + * {@link waitFor} repeatedly searches for a query to appear in the window until it is found or the timeout is reached + * @param searchInput A {@link WindowElementQuery} instance + * @param timeoutMs Timeout in milliseconds after which {@link waitFor} fails + * @param updateInterval Update interval in milliseconds to retry search + * @param params {@link OptionalSearchParameters} which are used to fine tune search + */ + public async waitFor( + searchInput: WindowElementQuery | Promise, + timeoutMs?: number, + updateInterval?: number, + params?: OptionalSearchParameters + ): Promise { + const needle = await searchInput; + + const timeoutValue = timeoutMs ?? 5000; + const updateIntervalValue = updateInterval ?? 500; + + this.providerRegistry + .getLogProvider() + .info( + `Waiting for ${needle.id} to appear in window. Timeout: ${ + timeoutValue / 1000 + } seconds, interval: ${updateIntervalValue} ms` + ); + return timeout( + updateIntervalValue, + timeoutValue, + () => { + return this.find(needle); + }, + { + signal: params?.abort + } + ); + } + + /** + * {@link on} registers a callback which is triggered once a certain searchInput image is found + * @param searchInput to trigger the callback on + * @param callback The {@link FindHookCallback} function to trigger + */ + public on(searchInput: WindowElementQuery, callback: WindowElementCallback): void { + const existingHooks = this.getHooksForInput(searchInput); + this.findHooks.set(searchInput, [...existingHooks, callback]); + this.providerRegistry + .getLogProvider() + .info( + `Registered callback for image ${searchInput.id}. There are currently ${ + existingHooks.length + 1 + } hooks registered` + ); + } + + private getHooksForInput( + input: WindowElementQuery + ): WindowElementCallback[] { + return this.findHooks.get(input) ?? []; + } } diff --git a/core/shared/lib/types/window.interface.ts b/core/shared/lib/types/window.interface.ts index 2e56c4e..e06aa68 100644 --- a/core/shared/lib/types/window.interface.ts +++ b/core/shared/lib/types/window.interface.ts @@ -1,4 +1,4 @@ -import { Point, Region, Size } from "../objects"; +import { OptionalSearchParameters, Point, Region, Size, WindowElementQuery } from "../objects"; import { WindowElement } from "./window-element.interface"; import { PointResultFindInput, RegionResultFindInput, WindowElementResultFindInput } from "./index"; @@ -22,6 +22,15 @@ export interface WindowInterface { findAll( searchInput: WindowElementResultFindInput | Promise ): Promise; + + waitFor( + searchInput: WindowElementQuery | Promise, + timeoutMs?: number, + updateInterval?: number, + params?: OptionalSearchParameters + ): Promise; + + on(searchInput: WindowElementQuery, callback: WindowElementCallback): void; } export type WindowedFindInput = From 8881668b258bfb49456b04f040750d6fcaeda0c1 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 01:36:20 +0200 Subject: [PATCH 10/12] Fixed broken tutorials link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index df45a66..c9e878e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Check out this demo video to get a first impression of what nut.js is capable of # Tutorials -Please consult the project website at [nutjs.dev](https://nutjs.dev/docs/tutorial-first_steps/prerequisites) for in-depth tutorials +Please consult the project website at [nutjs.dev](https://nutjs.dev/tutorials/first_steps#prerequisites) for in-depth tutorials # API Docs From 9b48ca09bc3ab7ef8883f3999cad13980872630f Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 02:05:21 +0200 Subject: [PATCH 11/12] Updated window tests --- core/nut.js/lib/window.class.spec.ts | 82 +++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/core/nut.js/lib/window.class.spec.ts b/core/nut.js/lib/window.class.spec.ts index 4a4056f..810a89f 100644 --- a/core/nut.js/lib/window.class.spec.ts +++ b/core/nut.js/lib/window.class.spec.ts @@ -7,10 +7,11 @@ import { WindowProviderInterface } from "@nut-tree/provider-interfaces"; import { mockPartial } from "sneer"; -import { Region, WindowElement, WindowElementDescription } from "@nut-tree/shared"; -import { windowElementDescribedBy } from "../index"; +import { Region, RGBA, WindowElement, WindowElementDescription } from "@nut-tree/shared"; +import { pixelWithColor, windowElementDescribedBy } from "../index"; import { NoopLogProvider } from "./provider/log/noop-log-provider.class"; +jest.setTimeout(50000); describe("Window class", () => { it("should retrieve the window region via provider", async () => { // GIVEN @@ -142,6 +143,83 @@ describe("Window class", () => { expect(elementInspectorMock).toHaveBeenCalledWith(mockWindowHandle, description); }); + describe("invalid input", () => { + it("should throw on invalid input to find", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock, + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description = new RGBA(255, 0, 255, 255); + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + const test = () => SUT.find(pixelWithColor(description) as any); + + // THEN + await expect(test).rejects.toThrowError(/.*'Error: Search input is not supported. Please use a valid search input type.'$/); + }); + + it("should throw on invalid input to findAll", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock, + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description = new RGBA(255, 0, 255, 255); + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + const test = () => SUT.findAll(pixelWithColor(description) as any); + + // THEN + await expect(test).rejects.toThrowError(/.*'Error: Search input is not supported. Please use a valid search input type.'$/); + }); + + it("should throw on invalid input to waitFor", async () => { + // GIVEN + const elementInspectorMock = jest.fn(); + const providerRegistryMock = mockPartial({ + getWindowElementInspector(): ElementInspectionProviderInterface { + return mockPartial({ + findElement: elementInspectorMock, + findElements: elementInspectorMock + }); + }, + getLogProvider(): LogProviderInterface { + return new NoopLogProvider(); + } + }); + const mockWindowHandle = 123; + const description = new RGBA(255, 0, 255, 255); + const SUT = new Window(providerRegistryMock, mockWindowHandle); + + // WHEN + const test = () => SUT.waitFor(pixelWithColor(description) as any, 100, 50); + + // THEN + await expect(test).rejects.toMatch(/.*'Error: Search input is not supported.*$/); + }); + }); + describe("hooks", () => { it("should trigger registered hooks", async () => { // GIVEN From 0f2939d2f71f4466ee9409294e4e047e47b5796a Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 2 Apr 2024 02:09:32 +0200 Subject: [PATCH 12/12] Removed leftover dead code --- core/nut.js/lib/window.class.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/nut.js/lib/window.class.ts b/core/nut.js/lib/window.class.ts index 24bc3cf..13a2381 100644 --- a/core/nut.js/lib/window.class.ts +++ b/core/nut.js/lib/window.class.ts @@ -135,7 +135,6 @@ export class Window implements WindowInterface { public async findAll( searchInput: WindowElementResultFindInput | Promise ): Promise { - // return this.providerRegistry.getWindowElementInspector().findElement(this.windowHandle, description); const needle = await searchInput; this.providerRegistry.getLogProvider().info(`Searching for ${needle} in window ${this.windowHandle}`);