mirror of
https://github.com/appium/appium.git
synced 2026-11-09 03:13:55 +08:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7b20335ea | ||
|
|
8b9597b0e5 | ||
|
|
556e90be7e | ||
|
|
ab8ee41e79 | ||
|
|
028a4274e1 | ||
|
|
9a4e5c77ac | ||
|
|
cafb9bf089 | ||
|
|
cdfe8ddc48 | ||
|
|
5ff00a0a5c | ||
|
|
fec5f3a377 | ||
|
|
5e5b76cf6b | ||
|
|
efd13358cc | ||
|
|
17eb3cac86 | ||
|
|
4419664ffd | ||
|
|
4be29e49b5 | ||
|
|
f115ebed49 | ||
|
|
beb9da3320 | ||
|
|
7c9d1363a9 | ||
|
|
5e28714442 | ||
|
|
14507371f7 | ||
|
|
384afc77ff | ||
|
|
77c9b58989 |
@@ -29,7 +29,7 @@ jobs:
|
||||
CROWDIN_PROJECT_ID: ${{ vars.CROWDIN_DOCS_PROJECT_ID }}
|
||||
CROWDIN_TOKEN: ${{ secrets.CROWDIN_DOCS_TOKEN }}
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v8.0.0
|
||||
uses: peter-evans/create-pull-request@v8.1.0
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
commit-message: 'docs(appium): Update documentation translations'
|
||||
|
||||
Generated
+453
-365
File diff suppressed because it is too large
Load Diff
+6
-6
@@ -86,7 +86,7 @@
|
||||
"devDependencies": {
|
||||
"@colors/colors": "1.6.0",
|
||||
"@eslint/js": "9.39.2",
|
||||
"@stylistic/eslint-plugin": "5.7.0",
|
||||
"@stylistic/eslint-plugin": "5.7.1",
|
||||
"@tsconfig/node20": "20.1.8",
|
||||
"@types/archiver": "7.0.0",
|
||||
"@types/argparse": "2.0.17",
|
||||
@@ -116,7 +116,7 @@
|
||||
"@types/ws": "8.18.1",
|
||||
"@types/xmldom": "0.1.34",
|
||||
"@types/yargs": "17.0.35",
|
||||
"asyncbox": "4.0.1",
|
||||
"asyncbox": "6.0.1",
|
||||
"chai": "6.2.2",
|
||||
"chai-as-promised": "8.0.2",
|
||||
"conventional-changelog-conventionalcommits": "7.0.2",
|
||||
@@ -139,19 +139,19 @@
|
||||
"mjpeg-server": "0.3.1",
|
||||
"mocha": "11.7.5",
|
||||
"npm-run-all2": "8.0.4",
|
||||
"prettier": "3.8.0",
|
||||
"prettier": "3.8.1",
|
||||
"rewiremock": "3.14.6",
|
||||
"rimraf": "6.1.2",
|
||||
"serve-static": "2.2.1",
|
||||
"sinon": "21.0.1",
|
||||
"sinon-chai": "4.0.1",
|
||||
"sync-monorepo-packages": "1.0.2",
|
||||
"teen_process": "4.0.7",
|
||||
"teen_process": "4.0.8",
|
||||
"ts-node": "10.9.2",
|
||||
"tsd": "0.33.0",
|
||||
"typescript": "5.9.3",
|
||||
"typescript-eslint": "8.53.0",
|
||||
"webdriverio": "9.23.0",
|
||||
"typescript-eslint": "8.53.1",
|
||||
"webdriverio": "9.23.2",
|
||||
"ws": "8.19.0",
|
||||
"yaml-js": "0.3.1"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,22 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [3.2.0](https://github.com/appium/appium/compare/appium@3.1.2...appium@3.2.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **appium:** Add repository links to verbose extensions list ([#21813](https://github.com/appium/appium/issues/21813)) ([ce91686](https://github.com/appium/appium/commit/ce91686852b0baab5f46990f5d40db4747922924))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **appium:** appium symlink behavior across drivers/plugins installations ([#21837](https://github.com/appium/appium/issues/21837)) ([529aeb4](https://github.com/appium/appium/commit/529aeb48f10419e21555cec05835ff07bb0e2e51))
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
* **docs:** add missing verbose flag for CLI list command ([#21814](https://github.com/appium/appium/issues/21814)) ([dcfaa88](https://github.com/appium/appium/commit/dcfaa88830f6a5c608c92f5ae72445e065261d38))
|
||||
|
||||
|
||||
|
||||
## [3.1.2](https://github.com/appium/appium/compare/appium@3.1.1...appium@3.1.2) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package appium
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const {asyncify} = require('asyncbox');
|
||||
|
||||
const appium = require('./build/lib/main.js');
|
||||
|
||||
if (require.main === module) {
|
||||
asyncify(appium.main);
|
||||
appium.main();
|
||||
}
|
||||
|
||||
module.exports = appium;
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
normalizeBasePath,
|
||||
} from '@appium/base-driver';
|
||||
import {util, env} from '@appium/support';
|
||||
import {asyncify} from 'asyncbox';
|
||||
import _ from 'lodash';
|
||||
import {AppiumDriver} from './appium';
|
||||
import {runExtensionCommand} from './cli/extension';
|
||||
@@ -490,7 +489,7 @@ async function main(args) {
|
||||
// (more specifically, `build/lib/main.js`)
|
||||
// the executable is now `../index.js`, so that module will typically be `require.main`.
|
||||
if (require.main === module) {
|
||||
asyncify(main);
|
||||
main();
|
||||
}
|
||||
|
||||
// everything below here is intended to be a public API.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "appium",
|
||||
"version": "3.1.2",
|
||||
"version": "3.2.0",
|
||||
"description": "Automation for Apps.",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -40,7 +40,8 @@
|
||||
"scripts/autoinstall-extensions.js",
|
||||
"types",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"build:docs": "node docs/scripts/build-docs.js",
|
||||
@@ -60,29 +61,28 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/base-driver": "^10.1.2",
|
||||
"@appium/base-plugin": "^3.0.5",
|
||||
"@appium/docutils": "^2.2.0",
|
||||
"@appium/logger": "^2.0.3",
|
||||
"@appium/schema": "^1.0.0",
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/types": "^1.1.2",
|
||||
"@appium/base-driver": "^10.2.0",
|
||||
"@appium/base-plugin": "^3.0.6",
|
||||
"@appium/docutils": "^2.2.1",
|
||||
"@appium/logger": "^2.0.4",
|
||||
"@appium/schema": "^1.0.1",
|
||||
"@appium/support": "^7.0.5",
|
||||
"@appium/types": "^1.2.0",
|
||||
"@sidvind/better-ajv-errors": "4.0.1",
|
||||
"ajv": "8.17.1",
|
||||
"ajv-formats": "3.0.1",
|
||||
"argparse": "2.0.1",
|
||||
"async-lock": "1.4.1",
|
||||
"asyncbox": "4.0.1",
|
||||
"axios": "1.13.2",
|
||||
"axios": "1.13.3",
|
||||
"bluebird": "3.7.2",
|
||||
"lilconfig": "3.1.3",
|
||||
"lodash": "4.17.21",
|
||||
"lru-cache": "11.2.4",
|
||||
"lodash": "4.17.23",
|
||||
"lru-cache": "11.2.5",
|
||||
"ora": "5.4.1",
|
||||
"package-changed": "3.0.0",
|
||||
"resolve-from": "5.0.0",
|
||||
"semver": "7.7.3",
|
||||
"teen_process": "4.0.7",
|
||||
"teen_process": "4.0.8",
|
||||
"type-fest": "5.4.1",
|
||||
"winston": "3.19.0",
|
||||
"ws": "8.19.0",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"devDependencies": {
|
||||
"webdriverio": "9.23.0"
|
||||
"webdriverio": "9.23.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ const yaml = require('yaml');
|
||||
const {fs, util, logger} = require('@appium/support');
|
||||
const Handlebars = require('handlebars');
|
||||
const _ = require('lodash');
|
||||
const {asyncify} = require('asyncbox');
|
||||
|
||||
const log = logger.getLogger('YamlParser');
|
||||
|
||||
@@ -417,4 +416,4 @@ async function main() {
|
||||
await generateCommandIndex();
|
||||
}
|
||||
|
||||
asyncify(main);
|
||||
main();
|
||||
|
||||
@@ -3,6 +3,20 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [10.2.0](https://github.com/appium/appium/compare/@appium/base-driver@10.1.2...@appium/base-driver@10.2.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **base-driver:** add ability to supply custom headers for proxy ([#21790](https://github.com/appium/appium/issues/21790)) ([a4b47c0](https://github.com/appium/appium/commit/a4b47c02c6e72381fa9d191d2a0bced106999705))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [10.1.2](https://github.com/appium/appium/compare/@appium/base-driver@10.1.1...@appium/base-driver@10.1.2) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/base-driver
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/base-driver",
|
||||
"version": "10.1.2",
|
||||
"version": "10.2.0",
|
||||
"description": "Base driver class for Appium drivers",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -34,7 +34,8 @@
|
||||
"static",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "run-p test:unit test:types",
|
||||
@@ -44,19 +45,19 @@
|
||||
"test:types": "tsd"
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/types": "^1.1.2",
|
||||
"@appium/support": "^7.0.5",
|
||||
"@appium/types": "^1.2.0",
|
||||
"@colors/colors": "1.6.0",
|
||||
"async-lock": "1.4.1",
|
||||
"asyncbox": "4.0.1",
|
||||
"axios": "1.13.2",
|
||||
"asyncbox": "6.0.1",
|
||||
"axios": "1.13.3",
|
||||
"bluebird": "3.7.2",
|
||||
"body-parser": "2.2.2",
|
||||
"express": "5.2.1",
|
||||
"fastest-levenshtein": "1.0.16",
|
||||
"http-status-codes": "2.3.0",
|
||||
"lodash": "4.17.21",
|
||||
"lru-cache": "11.2.4",
|
||||
"lodash": "4.17.23",
|
||||
"lru-cache": "11.2.5",
|
||||
"method-override": "3.0.0",
|
||||
"morgan": "1.10.1",
|
||||
"path-to-regexp": "8.3.0",
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [3.0.6](https://github.com/appium/appium/compare/@appium/base-plugin@3.0.5...@appium/base-plugin@3.0.6) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/base-plugin
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [3.0.5](https://github.com/appium/appium/compare/@appium/base-plugin@3.0.4...@appium/base-plugin@3.0.5) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/base-plugin
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/base-plugin",
|
||||
"version": "3.0.5",
|
||||
"version": "3.0.6",
|
||||
"description": "The base plugin used to create Appium plugins",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -32,7 +32,8 @@
|
||||
"build",
|
||||
"index.js",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -40,8 +41,8 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/base-driver": "^10.1.2",
|
||||
"@appium/support": "^7.0.4"
|
||||
"@appium/base-driver": "^10.2.0",
|
||||
"@appium/support": "^7.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0",
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [2.2.1](https://github.com/appium/appium/compare/@appium/docutils@2.2.0...@appium/docutils@2.2.1) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/docutils
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [2.2.0](https://github.com/appium/appium/compare/@appium/docutils@2.1.2...@appium/docutils@2.2.0) (2025-12-04)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/docutils",
|
||||
"version": "2.2.0",
|
||||
"version": "2.2.1",
|
||||
"description": "Documentation generation utilities for Appium and related projects",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -38,6 +38,7 @@
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test",
|
||||
"requirements.txt"
|
||||
],
|
||||
"scripts": {
|
||||
@@ -48,14 +49,14 @@
|
||||
"start": "node ./build/lib/cli/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/support": "^7.0.5",
|
||||
"consola": "3.4.2",
|
||||
"diff": "8.0.3",
|
||||
"lilconfig": "3.1.3",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"package-directory": "8.1.0",
|
||||
"read-pkg": "10.0.0",
|
||||
"teen_process": "4.0.7",
|
||||
"teen_process": "4.0.8",
|
||||
"type-fest": "5.4.1",
|
||||
"yaml": "2.8.2",
|
||||
"yargs": "18.0.0",
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.0.6](https://github.com/appium/appium/compare/@appium/driver-test-support@1.0.5...@appium/driver-test-support@1.0.6) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/driver-test-support
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [1.0.5](https://github.com/appium/appium/compare/@appium/driver-test-support@1.0.4...@appium/driver-test-support@1.0.5) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/driver-test-support
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/driver-test-support",
|
||||
"version": "1.0.5",
|
||||
"version": "1.0.6",
|
||||
"description": "Test utilities for Appium drivers",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -32,7 +32,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -41,13 +42,13 @@
|
||||
},
|
||||
"types": "./build/lib/index.d.ts",
|
||||
"dependencies": {
|
||||
"@appium/types": "^1.1.2",
|
||||
"axios": "1.13.2",
|
||||
"@appium/types": "^1.2.0",
|
||||
"axios": "1.13.3",
|
||||
"bluebird": "3.7.2",
|
||||
"chai": "6.2.2",
|
||||
"chai-as-promised": "8.0.2",
|
||||
"get-port": "7.1.0",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"sinon": "21.0.1",
|
||||
"type-fest": "5.4.1"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,20 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [3.0.0](https://github.com/appium/appium/compare/@appium/eslint-config-appium-ts@2.0.5...@appium/eslint-config-appium-ts@3.0.0) (2026-01-26)
|
||||
|
||||
|
||||
### ⚠ BREAKING CHANGES
|
||||
|
||||
* **eslint-config-appium-ts:** modernize ESLint config (#21781)
|
||||
|
||||
### Features
|
||||
|
||||
* **eslint-config-appium-ts:** add unicorn/prefer-node-protocol with warn ([#21864](https://github.com/appium/appium/issues/21864)) ([77c9b58](https://github.com/appium/appium/commit/77c9b5898907d7380276d5982c7e175d10440c73))
|
||||
* **eslint-config-appium-ts:** modernize ESLint config ([#21781](https://github.com/appium/appium/issues/21781)) ([b0b219a](https://github.com/appium/appium/commit/b0b219aa019d893bf30e90203a8265625777c47f))
|
||||
|
||||
|
||||
|
||||
## [2.0.5](https://github.com/appium/appium/compare/@appium/eslint-config-appium-ts@2.0.4...@appium/eslint-config-appium-ts@2.0.5) (2025-12-04)
|
||||
|
||||
|
||||
|
||||
@@ -148,7 +148,7 @@ export default defineConfig([
|
||||
* `return await somePromise` have their own use-cases.
|
||||
*/
|
||||
'require-await': 'off',
|
||||
'unicorn/prefer-node-protocol': 'error'
|
||||
'unicorn/prefer-node-protocol': 'warn'
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@appium/eslint-config-appium-ts",
|
||||
"type": "module",
|
||||
"version": "2.0.5",
|
||||
"version": "3.0.0",
|
||||
"description": "Shared ESLint config for Appium projects (TypeScript version)",
|
||||
"keywords": [
|
||||
"eslint",
|
||||
@@ -31,7 +31,7 @@
|
||||
"dependencies": {
|
||||
"@eslint/compat": "2.0.1",
|
||||
"@eslint/js": "9.39.2",
|
||||
"@stylistic/eslint-plugin": "5.7.0",
|
||||
"@stylistic/eslint-plugin": "5.7.1",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-import-resolver-typescript": "4.4.4",
|
||||
"eslint-plugin-import-x": "4.16.1",
|
||||
@@ -39,7 +39,7 @@
|
||||
"eslint-plugin-n": "17.23.2",
|
||||
"eslint-plugin-promise": "7.2.1",
|
||||
"eslint-plugin-unicorn": "62.0.0",
|
||||
"typescript-eslint": "8.53.0"
|
||||
"typescript-eslint": "8.53.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"eslint": "^9.0.0"
|
||||
|
||||
@@ -3,6 +3,20 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [5.1.0](https://github.com/appium/appium/compare/@appium/execute-driver-plugin@5.0.3...@appium/execute-driver-plugin@5.1.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **execute-driver-plugin:** Migrate to typescript ([#21887](https://github.com/appium/appium/issues/21887)) ([ab8ee41](https://github.com/appium/appium/commit/ab8ee41e79fcc4a2537d3873779f52f62607f097))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [5.0.3](https://github.com/appium/appium/compare/@appium/execute-driver-plugin@5.0.2...@appium/execute-driver-plugin@5.0.3) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/execute-driver-plugin
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
module.exports = require('./build/lib/plugin');
|
||||
+26
-54
@@ -2,12 +2,10 @@ import _ from 'lodash';
|
||||
import B from 'bluebird';
|
||||
import vm from 'node:vm';
|
||||
import {logger, util} from 'appium/support';
|
||||
import type {DriverScriptMessageEvent, ScriptResult, RunScriptResult} from './types';
|
||||
|
||||
const log = logger.getLogger('ExecuteDriver Child');
|
||||
/**
|
||||
* @type {(res: ScriptResult) => Promise<void>}
|
||||
*/
|
||||
let send;
|
||||
let send: (res: ScriptResult) => Promise<void>;
|
||||
|
||||
// duplicate defining these keys here so we don't need to re-load a huge appium
|
||||
// dependency tree into memory just to run a wdio script
|
||||
@@ -16,11 +14,11 @@ export const MJSONWP_ELEMENT_KEY = 'ELEMENT';
|
||||
|
||||
/**
|
||||
* Run the script in a VM.
|
||||
* @param {DriverScriptMessageEvent} eventParams
|
||||
* @returns {Promise<RunScriptResult>}
|
||||
* @param eventParams - The driver script message event parameters
|
||||
* @returns Promise resolving to the run script result
|
||||
* @throws {TypeError}
|
||||
*/
|
||||
async function runScript(eventParams) {
|
||||
async function runScript(eventParams: DriverScriptMessageEvent): Promise<RunScriptResult> {
|
||||
const {driverOpts, script, timeoutMs} = eventParams;
|
||||
if (!_.isNumber(timeoutMs)) {
|
||||
throw new TypeError('Timeout parameter must be a number');
|
||||
@@ -28,15 +26,17 @@ async function runScript(eventParams) {
|
||||
|
||||
/**
|
||||
* set up fake logger
|
||||
* @type {string[]}
|
||||
*/
|
||||
const logLevels = ['error', 'warn', 'log'];
|
||||
const logs = {};
|
||||
const consoleFns = {};
|
||||
for (const level of logLevels) {
|
||||
logs[level] = [];
|
||||
consoleFns[level] = (...logMsgs) => logs[level].push(...logMsgs);
|
||||
}
|
||||
const logs: {error: any[]; warn: any[]; log: any[]} = {
|
||||
error: [],
|
||||
warn: [],
|
||||
log: [],
|
||||
};
|
||||
const consoleFns: {error: (...args: any[]) => void; warn: (...args: any[]) => void; log: (...args: any[]) => void} = {
|
||||
error: (...logMsgs) => logs.error.push(...logMsgs),
|
||||
warn: (...logMsgs) => logs.warn.push(...logMsgs),
|
||||
log: (...logMsgs) => logs.log.push(...logMsgs),
|
||||
};
|
||||
|
||||
const {attach} = await import('webdriverio');
|
||||
|
||||
@@ -51,7 +51,7 @@ async function runScript(eventParams) {
|
||||
let result = await vm.runInNewContext(
|
||||
fullScript,
|
||||
{driver, console: consoleFns, Promise: B},
|
||||
{timeout: timeoutMs, breakOnSigint: true},
|
||||
{timeout: timeoutMs, breakOnSigint: true}
|
||||
);
|
||||
|
||||
result = coerceScriptResult(result);
|
||||
@@ -65,11 +65,10 @@ async function runScript(eventParams) {
|
||||
* response. So make sure we convert the things we know about to their
|
||||
* appropriate response format, and squash other weird things.
|
||||
*
|
||||
* @param {Object} obj - object to convert and sanitize
|
||||
*
|
||||
* @return {Object} - safely converted object
|
||||
* @param obj - object to convert and sanitize
|
||||
* @return safely converted object
|
||||
*/
|
||||
function coerceScriptResult(obj) {
|
||||
function coerceScriptResult(obj: any): any {
|
||||
// first ensure obj is of a type that can be JSON encoded safely. This will
|
||||
// get rid of custom objects, functions, etc... and turn them into POJOs
|
||||
try {
|
||||
@@ -77,12 +76,12 @@ function coerceScriptResult(obj) {
|
||||
} catch {
|
||||
log.warn(
|
||||
'Could not convert executeDriverScript to safe response!' +
|
||||
`Result was: ${JSON.stringify(obj)}. Will make it null`,
|
||||
`Result was: ${JSON.stringify(obj)}. Will make it null`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
let res;
|
||||
let res: any;
|
||||
|
||||
// now we begin our recursive case options
|
||||
if (_.isPlainObject(obj)) {
|
||||
@@ -125,19 +124,18 @@ function coerceScriptResult(obj) {
|
||||
|
||||
/**
|
||||
* Entry point to runScript
|
||||
* @param {DriverScriptMessageEvent} eventParams
|
||||
* @param eventParams - The driver script message event parameters
|
||||
*/
|
||||
async function main(eventParams) {
|
||||
async function main(eventParams: DriverScriptMessageEvent): Promise<void> {
|
||||
/**
|
||||
* keep the response of runScript
|
||||
* @type {ScriptResult}
|
||||
*/
|
||||
let res;
|
||||
let res: ScriptResult;
|
||||
log.info('Parameters received from parent process');
|
||||
try {
|
||||
res = {success: await runScript(eventParams)};
|
||||
log.info('runScript success');
|
||||
} catch (error) {
|
||||
} catch (error: any) {
|
||||
log.info('runScript error');
|
||||
res = {error: {message: error.message, stack: error.stack}};
|
||||
}
|
||||
@@ -146,33 +144,7 @@ async function main(eventParams) {
|
||||
|
||||
// ensure we're running this script in IPC mode
|
||||
if (require.main === module && _.isFunction(process.send)) {
|
||||
send = B.promisify(process.send, {context: process});
|
||||
send = B.promisify(process.send, {context: process}) as (res: ScriptResult) => Promise<void>;
|
||||
log.info('Running driver execution in child process');
|
||||
process.on('message', main);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @typedef DriverScriptMessageEvent
|
||||
* @property {any} driverOpts - the driver options
|
||||
* @property {string} script - the javascript to execute
|
||||
* @property {number} timeoutMs - script timeout in milliseconds
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef ScriptResult
|
||||
* @property {any} [success]
|
||||
* @property {ScriptResultError} [error]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef ScriptResultError
|
||||
* @property {any} message
|
||||
* @property {any} stack
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef RunScriptResult
|
||||
* @property {any} result
|
||||
* @property {object} logs
|
||||
*/
|
||||
@@ -0,0 +1,3 @@
|
||||
export {ExecuteDriverPlugin} from './plugin';
|
||||
export {W3C_ELEMENT_KEY, MJSONWP_ELEMENT_KEY} from './execute-child';
|
||||
export type * from './types';
|
||||
+31
-32
@@ -3,45 +3,46 @@ import _ from 'lodash';
|
||||
import cp from 'node:child_process';
|
||||
import {timing} from 'appium/support';
|
||||
import B from 'bluebird';
|
||||
import type {ExternalDriver, NextPluginCallback, MethodMap, PluginCommand} from '@appium/types';
|
||||
|
||||
const FEAT_FLAG = 'execute_driver_script';
|
||||
const DEFAULT_SCRIPT_TIMEOUT_MS = 1000 * 60 * 60; // default to 1 hour timeout
|
||||
const SCRIPT_TYPE_WDIO = 'webdriverio';
|
||||
|
||||
export default class ExecuteDriverPlugin extends BasePlugin {
|
||||
static newMethodMap = /** @type {const} */ ({
|
||||
export class ExecuteDriverPlugin extends BasePlugin {
|
||||
static newMethodMap: MethodMap<ExecuteDriverPlugin> = {
|
||||
'/session/:sessionId/appium/execute_driver': {
|
||||
POST: {
|
||||
command: 'executeDriverScript',
|
||||
payloadParams: {required: ['script'], optional: ['type', 'timeout']},
|
||||
},
|
||||
},
|
||||
});
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* This method takes a string which is executed as javascript in the context of
|
||||
* a new nodejs VM, and which has available a webdriverio driver object, having
|
||||
* already been attached to the currently running session.
|
||||
*
|
||||
* @param {import('@appium/types').NextPluginCallback} next - standard behaviour for executeDriverScript
|
||||
* @param {import('@appium/types').ExternalDriver} driver - Appium driver handling this command
|
||||
* @param {string} script - the string representing the driver script to run
|
||||
* @param {string} [scriptType='webdriverio'] - the name of the driver script
|
||||
* library (currently only webdriverio is supported)
|
||||
* @param {number} [timeoutMs=3600000] - timeout for the script process
|
||||
*
|
||||
* @returns {Promise<any>} - a JSONifiable object representing the return value of
|
||||
* the script
|
||||
* @type {import('@appium/types').PluginCommand<import('@appium/types').ExternalDriver, [script: string, scriptType: string?, timeoutMs: number?]>}
|
||||
* @param next - standard behaviour for executeDriverScript
|
||||
* @param driver - Appium driver handling this command
|
||||
* @param script - the string representing the driver script to run
|
||||
* @param scriptType - the name of the driver script library (currently only webdriverio is supported). Defaults to `'webdriverio'`.
|
||||
* @param timeoutMs - timeout for the script process. Defaults to `3600000`.
|
||||
* @returns a JSONifiable object representing the return value of the script
|
||||
* @throws {Error}
|
||||
*/
|
||||
async executeDriverScript(
|
||||
next,
|
||||
driver,
|
||||
script,
|
||||
scriptType = 'webdriverio',
|
||||
timeoutMs = DEFAULT_SCRIPT_TIMEOUT_MS
|
||||
) {
|
||||
executeDriverScript: PluginCommand<
|
||||
ExternalDriver,
|
||||
[script: string, scriptType?: string, timeoutMs?: number],
|
||||
any
|
||||
> = async (
|
||||
next: NextPluginCallback,
|
||||
driver: ExternalDriver,
|
||||
script: string,
|
||||
scriptType: string = 'webdriverio',
|
||||
timeoutMs: number = DEFAULT_SCRIPT_TIMEOUT_MS
|
||||
) => {
|
||||
if (!driver.isFeatureEnabled(FEAT_FLAG)) {
|
||||
throw new Error(
|
||||
`Execute driver script functionality is not available ` +
|
||||
@@ -80,13 +81,13 @@ export default class ExecuteDriverPlugin extends BasePlugin {
|
||||
isMobile: true,
|
||||
capabilities: driver.caps,
|
||||
};
|
||||
this.logger.info(
|
||||
this.log.info(
|
||||
`Constructed webdriverio driver options; W3C mode is ${driverOpts.isW3C ? 'on' : 'off'}`
|
||||
);
|
||||
|
||||
// fork the execution script as a child process
|
||||
const childScript = require.resolve('./execute-child.js');
|
||||
this.logger.info(`Forking process to run webdriver script as child using ${childScript}`);
|
||||
this.log.info(`Forking process to run webdriver script as child using ${childScript}`);
|
||||
const scriptProc = cp.fork(childScript);
|
||||
|
||||
// keep track of whether we have canceled the script timeout, so we can stop
|
||||
@@ -99,11 +100,11 @@ export default class ExecuteDriverPlugin extends BasePlugin {
|
||||
|
||||
// promise that deals with the result from the child process
|
||||
const waitForResult = async () => {
|
||||
const res = await new B((res) => {
|
||||
const res = await new B<{error?: {message: string}; success?: any}>((res) => {
|
||||
scriptProc.on('message', res); // this is node IPC
|
||||
});
|
||||
|
||||
this.logger.info(
|
||||
this.log.info(
|
||||
'Received execute driver script result from child process, shutting it down'
|
||||
);
|
||||
|
||||
@@ -134,12 +135,12 @@ export default class ExecuteDriverPlugin extends BasePlugin {
|
||||
|
||||
// now that the child script is alive, send it the data it needs to start
|
||||
// running the driver script
|
||||
this.logger.info('Sending driver and script data to child');
|
||||
this.log.info('Sending driver and script data to child');
|
||||
scriptProc.send({driverOpts, script, timeoutMs});
|
||||
|
||||
// and set up a race between the response from the child and the timeout
|
||||
return await B.race([waitForResult(), waitForTimeout()]);
|
||||
} catch (err) {
|
||||
} catch (err: any) {
|
||||
throw new Error(`Could not execute driver script. Original error was: ${err}`);
|
||||
} finally {
|
||||
// ensure we always cancel the timeout so that the timeout promise stops
|
||||
@@ -147,18 +148,16 @@ export default class ExecuteDriverPlugin extends BasePlugin {
|
||||
timeoutCanceled = true;
|
||||
|
||||
if (scriptProc.connected) {
|
||||
this.logger.info('Disconnecting from child proc');
|
||||
this.log.info('Disconnecting from child proc');
|
||||
scriptProc.disconnect();
|
||||
}
|
||||
|
||||
if (scriptProc.exitCode === null) {
|
||||
this.logger.info('Disconnecting from and killing driver script child proc');
|
||||
this.log.info('Disconnecting from and killing driver script child proc');
|
||||
scriptProc.kill();
|
||||
} else {
|
||||
this.logger.info('Script already ended on its own, no need to kill it');
|
||||
this.log.info('Script already ended on its own, no need to kill it');
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export {ExecuteDriverPlugin};
|
||||
@@ -0,0 +1,24 @@
|
||||
export interface DriverScriptMessageEvent {
|
||||
driverOpts: any;
|
||||
script: string;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface ScriptResultError {
|
||||
message: any;
|
||||
stack: any;
|
||||
}
|
||||
|
||||
export interface ScriptResult {
|
||||
success?: RunScriptResult;
|
||||
error?: ScriptResultError;
|
||||
}
|
||||
|
||||
export interface RunScriptResult {
|
||||
result: any;
|
||||
logs: {
|
||||
error: any[];
|
||||
warn: any[];
|
||||
log: any[];
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/execute-driver-plugin",
|
||||
"version": "5.0.3",
|
||||
"version": "5.1.0",
|
||||
"description": "Plugin for batching and executing Appium driver commands",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -23,24 +23,25 @@
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"author": "https://github.com/appium",
|
||||
"types": "./build/lib/plugin.d.ts",
|
||||
"main": "./build/lib/index.js",
|
||||
"types": "./build/lib/index.d.ts",
|
||||
"files": [
|
||||
"build",
|
||||
"lib",
|
||||
"index.js",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
"test:e2e": "mocha --exit -t 160s --slow 20s \"./test/e2e/**/*.spec.js\"",
|
||||
"test:smoke": "node ./index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
"test:e2e": "mocha --exit -t 160s --slow 20s \"./test/e2e/**/*.spec.*ts\"",
|
||||
"test:smoke": "node ./build/lib/index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.*ts\""
|
||||
},
|
||||
"dependencies": {
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21",
|
||||
"webdriverio": "9.23.0"
|
||||
"lodash": "4.17.23",
|
||||
"webdriverio": "9.23.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"appium": "^3.0.0-beta.0"
|
||||
|
||||
+30
-32
@@ -1,11 +1,13 @@
|
||||
// @ts-check
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
import {pluginE2EHarness} from '@appium/plugin-test-support';
|
||||
import {remote as wdio} from 'webdriverio';
|
||||
import {W3C_ELEMENT_KEY, MJSONWP_ELEMENT_KEY} from '../../lib/execute-child';
|
||||
import {fs} from 'appium/support';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
const THIS_PLUGIN_DIR = path.join(__dirname, '..', '..');
|
||||
const APPIUM_HOME = path.join(THIS_PLUGIN_DIR, 'local_appium_home');
|
||||
@@ -34,22 +36,18 @@ const WDIO_OPTS = {
|
||||
};
|
||||
|
||||
describe('ExecuteDriverPlugin', function () {
|
||||
/** @type {import('webdriverio').Browser<'async'>} */
|
||||
let driver;
|
||||
let driver: any;
|
||||
|
||||
const basicScript = `return 'foo'`;
|
||||
/**
|
||||
* @type {import('@appium/plugin-test-support').E2ESetupOpts}
|
||||
*/
|
||||
const e2eSetupOpts = {
|
||||
before,
|
||||
after,
|
||||
host: TEST_HOST,
|
||||
driverName: 'fake',
|
||||
driverSource: 'local',
|
||||
driverSource: 'local' as const,
|
||||
driverSpec: FAKE_DRIVER_DIR,
|
||||
pluginName: 'execute-driver',
|
||||
pluginSource: 'local',
|
||||
pluginSource: 'local' as const,
|
||||
pluginSpec: THIS_PLUGIN_DIR,
|
||||
appiumHome: APPIUM_HOME,
|
||||
};
|
||||
@@ -66,9 +64,9 @@ describe('ExecuteDriverPlugin', function () {
|
||||
|
||||
it('should not work unless the allowInsecure feature flag is set', async function () {
|
||||
driver = await wdio({...WDIO_OPTS, port: this.port});
|
||||
await driver
|
||||
.executeDriverScript(basicScript)
|
||||
.should.be.rejectedWith(/allow-insecure.+execute_driver_script/i);
|
||||
await expect(driver.executeDriverScript(basicScript)).to.eventually.be.rejectedWith(
|
||||
/allow-insecure.+execute_driver_script/i
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -92,15 +90,15 @@ describe('ExecuteDriverPlugin', function () {
|
||||
`;
|
||||
const expectedTimeouts = {command: 60000, implicit: 0};
|
||||
const {result, logs} = await driver.executeDriverScript(script);
|
||||
result[0].should.eql(expectedTimeouts);
|
||||
result[1].build.should.exist;
|
||||
result[1].build.version.should.exist;
|
||||
logs.should.eql({error: [], warn: [], log: []});
|
||||
expect(result[0]).to.eql(expectedTimeouts);
|
||||
expect(result[1].build).to.exist;
|
||||
expect(result[1].build.version).to.exist;
|
||||
expect(logs).to.eql({error: [], warn: [], log: []});
|
||||
});
|
||||
|
||||
it('should fail with any script type other than webdriverio currently', async function () {
|
||||
const script = `return 'foo'`;
|
||||
await driver.executeDriverScript(script, 'wd').should.be.rejectedWith(/webdriverio/);
|
||||
await expect(driver.executeDriverScript(script, 'wd')).to.eventually.be.rejectedWith(/webdriverio/);
|
||||
});
|
||||
|
||||
it('should execute a webdriverio script that returns elements correctly', async function () {
|
||||
@@ -108,7 +106,7 @@ describe('ExecuteDriverPlugin', function () {
|
||||
return await driver.$("#Button1");
|
||||
`;
|
||||
const {result} = await driver.executeDriverScript(script);
|
||||
result.should.eql({
|
||||
expect(result).to.eql({
|
||||
[W3C_ELEMENT_KEY]: '1',
|
||||
[MJSONWP_ELEMENT_KEY]: '1',
|
||||
});
|
||||
@@ -124,7 +122,7 @@ describe('ExecuteDriverPlugin', function () {
|
||||
[W3C_ELEMENT_KEY]: '1',
|
||||
[MJSONWP_ELEMENT_KEY]: '1',
|
||||
};
|
||||
result.should.eql({element: elObj, elements: [elObj, elObj]});
|
||||
expect(result).to.eql({element: elObj, elements: [elObj, elObj]});
|
||||
});
|
||||
|
||||
it('should store and return logs to the user', async function () {
|
||||
@@ -136,7 +134,7 @@ describe('ExecuteDriverPlugin', function () {
|
||||
return null;
|
||||
`;
|
||||
const {logs} = await driver.executeDriverScript(script);
|
||||
logs.should.eql({log: ['foo', 'foo2'], warn: ['bar'], error: ['baz']});
|
||||
expect(logs).to.eql({log: ['foo', 'foo2'], warn: ['bar'], error: ['baz']});
|
||||
});
|
||||
|
||||
it('should have appium specific commands available', async function () {
|
||||
@@ -144,7 +142,7 @@ describe('ExecuteDriverPlugin', function () {
|
||||
return typeof driver.lock;
|
||||
`;
|
||||
const {result} = await driver.executeDriverScript(script);
|
||||
result.should.eql('function');
|
||||
expect(result).to.eql('function');
|
||||
});
|
||||
|
||||
it('should correctly handle errors that happen in a webdriverio script', async function () {
|
||||
@@ -152,20 +150,20 @@ describe('ExecuteDriverPlugin', function () {
|
||||
return await driver.$("~notfound");
|
||||
`;
|
||||
const {result} = await driver.executeDriverScript(script);
|
||||
result.error.error.should.equal('no such element');
|
||||
result.error.message.should.match(/element could not be located/);
|
||||
result.error.stacktrace.should.includes('NoSuchElementError:');
|
||||
result.selector.should.equal('~notfound');
|
||||
result.sessionId.should.equal(driver.sessionId);
|
||||
expect(result.error.error).to.equal('no such element');
|
||||
expect(result.error.message).to.match(/element could not be located/);
|
||||
expect(result.error.stacktrace).to.include('NoSuchElementError:');
|
||||
expect(result.selector).to.equal('~notfound');
|
||||
expect(result.sessionId).to.equal(driver.sessionId);
|
||||
});
|
||||
|
||||
it('should correctly handle errors that happen when a script cannot be compiled', async function () {
|
||||
const script = `
|
||||
return {;
|
||||
`;
|
||||
await driver
|
||||
.executeDriverScript(script)
|
||||
.should.be.rejectedWith(/Could not execute driver script.+Unexpected token/);
|
||||
await expect(driver.executeDriverScript(script)).to.eventually.be.rejectedWith(
|
||||
/Could not execute driver script.+Unexpected token/
|
||||
);
|
||||
});
|
||||
|
||||
it('should be able to set a timeout on a driver script', async function () {
|
||||
@@ -173,9 +171,9 @@ describe('ExecuteDriverPlugin', function () {
|
||||
await Promise.delay(1000);
|
||||
return true;
|
||||
`;
|
||||
await driver
|
||||
.executeDriverScript(script, 'webdriverio', 50)
|
||||
.should.be.rejectedWith(/.+50.+timeout.+/);
|
||||
await expect(driver.executeDriverScript(script, 'webdriverio', 50)).to.eventually.be.rejectedWith(
|
||||
/.+50.+timeout.+/
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
import {ExecuteDriverPlugin} from '../../lib/plugin';
|
||||
|
||||
describe('execute driver plugin', function () {
|
||||
let should;
|
||||
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
should = chai.should();
|
||||
});
|
||||
|
||||
it('should exist', function () {
|
||||
should.exist(ExecuteDriverPlugin);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import {ExecuteDriverPlugin} from '../../lib/plugin';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
describe('execute driver plugin', function () {
|
||||
it('should exist', function () {
|
||||
expect(ExecuteDriverPlugin).to.exist;
|
||||
});
|
||||
});
|
||||
@@ -2,8 +2,9 @@
|
||||
"compilerOptions": {
|
||||
"outDir": "build",
|
||||
"checkJs": true,
|
||||
"esModuleInterop": true
|
||||
"esModuleInterop": true,
|
||||
"types": ["mocha", "chai", "chai-as-promised"]
|
||||
},
|
||||
"extends": "@appium/tsconfig/tsconfig.plugin.json",
|
||||
"include": ["lib"]
|
||||
"include": ["lib", "test"]
|
||||
}
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [6.0.2](https://github.com/appium/appium/compare/@appium/fake-driver@6.0.1...@appium/fake-driver@6.0.2) (2026-01-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [6.0.1](https://github.com/appium/appium/compare/@appium/fake-driver@6.0.0...@appium/fake-driver@6.0.1) (2025-10-08)
|
||||
|
||||
**Note:** Version bump only for package @appium/fake-driver
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/fake-driver",
|
||||
"version": "6.0.1",
|
||||
"version": "6.0.2",
|
||||
"description": "Mock driver used internally by Appium for testing. Ignore",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -31,7 +31,8 @@
|
||||
"build",
|
||||
"test/fixtures",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "cpy lib/screen.png build",
|
||||
@@ -43,9 +44,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@xmldom/xmldom": "0.9.8",
|
||||
"asyncbox": "4.0.1",
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"xpath": "0.0.34"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [4.0.6](https://github.com/appium/appium/compare/@appium/fake-plugin@4.0.5...@appium/fake-plugin@4.0.6) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/fake-plugin
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [4.0.5](https://github.com/appium/appium/compare/@appium/fake-plugin@4.0.4...@appium/fake-plugin@4.0.5) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/fake-plugin
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/fake-plugin",
|
||||
"version": "4.0.5",
|
||||
"version": "4.0.6",
|
||||
"description": "A fake Appium plugin",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -31,7 +31,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -39,10 +40,10 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/base-plugin": "^3.0.5",
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/base-plugin": "^3.0.6",
|
||||
"@appium/support": "^7.0.5",
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21"
|
||||
"lodash": "4.17.23"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"appium": "^3.0.0-beta.0"
|
||||
|
||||
@@ -3,6 +3,20 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [4.1.0](https://github.com/appium/appium/compare/@appium/images-plugin@4.0.4...@appium/images-plugin@4.1.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **images-plugin:** Migrate to typescript ([#21865](https://github.com/appium/appium/issues/21865)) ([1450737](https://github.com/appium/appium/commit/14507371f7cb3c5e468f58ebc3b303109dcff3d7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [4.0.4](https://github.com/appium/appium/compare/@appium/images-plugin@4.0.3...@appium/images-plugin@4.0.4) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/images-plugin
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
module.exports = require('./build/lib/plugin');
|
||||
@@ -1,96 +0,0 @@
|
||||
import _ from 'lodash';
|
||||
import {errors} from 'appium/driver';
|
||||
import {getImagesMatches, getImagesSimilarity, getImageOccurrence} from '@appium/opencv';
|
||||
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE} from './constants';
|
||||
|
||||
/**
|
||||
* @typedef {{visualization: string|null|undefined}} Visualized
|
||||
* @typedef {import('@appium/opencv').MatchingResult & Visualized} MatchingResult
|
||||
* @typedef {import('@appium/opencv').OccurrenceResult & Visualized} OccurrenceResult
|
||||
* @typedef {import('@appium/opencv').SimilarityResult & Visualized} SimilarityResult
|
||||
* @typedef {MatchingResult|OccurrenceResult|SimilarityResult|SimilarityResult[]} ComparisonResult
|
||||
*/
|
||||
|
||||
/**
|
||||
* Performs images comparison using OpenCV framework features.
|
||||
* It is expected that both OpenCV framework and opencv4nodejs
|
||||
* module are installed on the machine where Appium server is running.
|
||||
*
|
||||
* @param {string} mode - One of possible comparison modes:
|
||||
* matchFeatures, getSimilarity, matchTemplate
|
||||
* @param {string|Buffer} firstImage - Base64-encoded image file.
|
||||
* All image formats, that OpenCV library itself accepts, are supported.
|
||||
* @param {string|Buffer} secondImage - Base64-encoded image file.
|
||||
* All image formats, that OpenCV library itself accepts, are supported.
|
||||
* @param {import('@appium/opencv').MatchingOptions
|
||||
* |import('@appium/opencv').SimilarityOptions
|
||||
* |import('@appium/opencv').OccurrenceOptions} [options={}] - The content of this dictionary depends
|
||||
* on the actual `mode` value. See the documentation on `@appium/support`
|
||||
* module for more details.
|
||||
* @returns {Promise<ComparisonResult>} The content of the resulting dictionary depends
|
||||
* on the actual `mode` and `options` values. See the documentation on
|
||||
* `@appium/support` module for more details.
|
||||
* @throws {Error} If required OpenCV modules are not installed or
|
||||
* if `mode` value is incorrect or if there was an unexpected issue while
|
||||
* matching the images.
|
||||
*/
|
||||
async function compareImages(mode, firstImage, secondImage, options = {}) {
|
||||
const img1 = Buffer.isBuffer(firstImage) ? firstImage : Buffer.from(firstImage, 'base64');
|
||||
const img2 = Buffer.isBuffer(secondImage) ? secondImage : Buffer.from(secondImage, 'base64');
|
||||
let result;
|
||||
switch (_.toLower(mode)) {
|
||||
case MATCH_FEATURES_MODE.toLowerCase():
|
||||
try {
|
||||
result = await getImagesMatches(
|
||||
img1, img2, /** @type {import('@appium/opencv').MatchingOptions} */(options)
|
||||
);
|
||||
} catch {
|
||||
// might throw if no matches
|
||||
result = /** @type {import('@appium/opencv').MatchingResult} */({count: 0});
|
||||
}
|
||||
break;
|
||||
case GET_SIMILARITY_MODE.toLowerCase():
|
||||
result = await getImagesSimilarity(
|
||||
img1, img2, /** @type {import('@appium/opencv').SimilarityOptions} */(options));
|
||||
break;
|
||||
case MATCH_TEMPLATE_MODE.toLowerCase(): {
|
||||
const opts = /** @type {import('@appium/opencv').OccurrenceOptions} */(options);
|
||||
// firstImage/img1 is the full image and secondImage/img2 is the partial one
|
||||
result = await getImageOccurrence(
|
||||
img1, img2, opts
|
||||
);
|
||||
if (opts.multiple && result.multiple) {
|
||||
return result.multiple.map(convertVisualizationToBase64);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new errors.InvalidArgumentError(
|
||||
`'${mode}' images comparison mode is unknown. ` +
|
||||
`Only ${JSON.stringify([
|
||||
MATCH_FEATURES_MODE,
|
||||
GET_SIMILARITY_MODE,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
])} modes are supported.`
|
||||
);
|
||||
}
|
||||
return convertVisualizationToBase64(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* base64 encodes the visualization part of the result
|
||||
* (if necessary)
|
||||
*
|
||||
* @param {Partial<{visualization: Buffer|null}>} element - occurrence result
|
||||
* @returns {any} I know this looks ugly from the typing perspective
|
||||
**/
|
||||
function convertVisualizationToBase64(element) {
|
||||
return Buffer.isBuffer(element.visualization)
|
||||
? {
|
||||
...(element),
|
||||
visualization: element.visualization.toString('base64')
|
||||
}
|
||||
: element;
|
||||
}
|
||||
|
||||
export {compareImages};
|
||||
@@ -0,0 +1,100 @@
|
||||
import _ from 'lodash';
|
||||
import {errors} from 'appium/driver';
|
||||
import {
|
||||
getImagesMatches,
|
||||
getImagesSimilarity,
|
||||
getImageOccurrence,
|
||||
type MatchingResult,
|
||||
type OccurrenceResult,
|
||||
type SimilarityResult,
|
||||
type MatchingOptions,
|
||||
type SimilarityOptions,
|
||||
type OccurrenceOptions,
|
||||
} from '@appium/opencv';
|
||||
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE} from './constants';
|
||||
import type {ComparisonResult} from './types';
|
||||
|
||||
/**
|
||||
* Performs images comparison using OpenCV framework features.
|
||||
* It is expected that both OpenCV framework and opencv4nodejs
|
||||
* module are installed on the machine where Appium server is running.
|
||||
*
|
||||
* @param mode - One of possible comparison modes:
|
||||
* matchFeatures, getSimilarity, matchTemplate
|
||||
* @param firstImage - Base64-encoded image file.
|
||||
* All image formats, that OpenCV library itself accepts, are supported.
|
||||
* @param secondImage - Base64-encoded image file.
|
||||
* All image formats, that OpenCV library itself accepts, are supported.
|
||||
* @param options - The content of this dictionary depends
|
||||
* on the actual `mode` value. See the documentation on `@appium/support`
|
||||
* module for more details.
|
||||
* @returns The content of the resulting dictionary depends
|
||||
* on the actual `mode` and `options` values. See the documentation on
|
||||
* `@appium/support` module for more details.
|
||||
* @throws {Error} If required OpenCV modules are not installed or
|
||||
* if `mode` value is incorrect or if there was an unexpected issue while
|
||||
* matching the images.
|
||||
*/
|
||||
export async function compareImages(
|
||||
mode: string,
|
||||
firstImage: string | Buffer,
|
||||
secondImage: string | Buffer,
|
||||
options: MatchingOptions | SimilarityOptions | OccurrenceOptions = {}
|
||||
): Promise<ComparisonResult> {
|
||||
const img1 = Buffer.isBuffer(firstImage) ? firstImage : Buffer.from(firstImage, 'base64');
|
||||
const img2 = Buffer.isBuffer(secondImage) ? secondImage : Buffer.from(secondImage, 'base64');
|
||||
let result: MatchingResult | SimilarityResult | OccurrenceResult;
|
||||
switch (_.toLower(mode)) {
|
||||
case MATCH_FEATURES_MODE.toLowerCase():
|
||||
try {
|
||||
result = await getImagesMatches(img1, img2, options as MatchingOptions);
|
||||
} catch {
|
||||
// might throw if no matches
|
||||
result = {count: 0} as MatchingResult;
|
||||
}
|
||||
break;
|
||||
case GET_SIMILARITY_MODE.toLowerCase():
|
||||
result = await getImagesSimilarity(img1, img2, options as SimilarityOptions);
|
||||
break;
|
||||
case MATCH_TEMPLATE_MODE.toLowerCase(): {
|
||||
const opts = options as OccurrenceOptions;
|
||||
// firstImage/img1 is the full image and secondImage/img2 is the partial one
|
||||
result = await getImageOccurrence(img1, img2, opts);
|
||||
if (opts.multiple && (result as OccurrenceResult).multiple) {
|
||||
const multipleResults = (result as OccurrenceResult).multiple;
|
||||
if (multipleResults) {
|
||||
return multipleResults.map(convertVisualizationToBase64);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new errors.InvalidArgumentError(
|
||||
`'${mode}' images comparison mode is unknown. ` +
|
||||
`Only ${JSON.stringify([
|
||||
MATCH_FEATURES_MODE,
|
||||
GET_SIMILARITY_MODE,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
])} modes are supported.`
|
||||
);
|
||||
}
|
||||
return convertVisualizationToBase64(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* base64 encodes the visualization part of the result
|
||||
* (if necessary)
|
||||
*
|
||||
* @param element - occurrence result
|
||||
* @returns result with base64-encoded visualization
|
||||
**/
|
||||
function convertVisualizationToBase64(
|
||||
element: Partial<{visualization: Buffer | null}>
|
||||
): any {
|
||||
return Buffer.isBuffer(element.visualization)
|
||||
? {
|
||||
...element,
|
||||
visualization: element.visualization.toString('base64'),
|
||||
}
|
||||
: element;
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
import {node} from 'appium/support';
|
||||
|
||||
export const IMAGE_STRATEGY = '-image';
|
||||
|
||||
export const IMAGE_ELEMENT_PREFIX = 'appium-image-element-';
|
||||
export const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions';
|
||||
export const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions';
|
||||
export const IMAGE_TAP_STRATEGIES = [IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C];
|
||||
export const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
|
||||
|
||||
export const MATCH_FEATURES_MODE = 'matchFeatures';
|
||||
export const GET_SIMILARITY_MODE = 'getSimilarity';
|
||||
export const MATCH_TEMPLATE_MODE = 'matchTemplate';
|
||||
|
||||
export const DEFAULT_MATCH_THRESHOLD = 0.4;
|
||||
|
||||
export const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
|
||||
|
||||
export const DEFAULT_SETTINGS = node.deepFreeze({
|
||||
// value between 0 and 1 representing match strength, below which an image
|
||||
// element will not be found
|
||||
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
|
||||
|
||||
// One of possible image matching methods.
|
||||
// Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
// for more details.
|
||||
// TM_CCOEFF_NORMED by default
|
||||
imageMatchMethod: '',
|
||||
|
||||
// if the image returned by getScreenshot differs in size or aspect ratio
|
||||
// from the screen, attempt to fix it automatically
|
||||
fixImageFindScreenshotDims: true,
|
||||
|
||||
// whether Appium should ensure that an image template sent in during image
|
||||
// element find should have its size adjusted so the match algorithm will not
|
||||
// complain
|
||||
fixImageTemplateSize: false,
|
||||
|
||||
// whether Appium should ensure that an image template sent in during image
|
||||
// element find should have its scale adjusted to display size so the match
|
||||
// algorithm will not complain.
|
||||
// e.g. iOS has `width=375, height=667` window rect, but its screenshot is
|
||||
// `width=750 × height=1334` pixels. This setting help to adjust the scale
|
||||
// if a user use `width=750 × height=1334` pixels's base template image.
|
||||
fixImageTemplateScale: false,
|
||||
|
||||
// Users might have scaled template image to reduce their storage size.
|
||||
// This setting allows users to scale a template image they send to Appium server
|
||||
// so that the Appium server compares the actual scale users originally had.
|
||||
// e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
|
||||
// the user can set {defaultImageTemplateScale: 4.0} to scale the small image
|
||||
// to the original one so that Appium can compare it as the original one.
|
||||
defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
|
||||
// whether Appium should re-check that an image element can be matched
|
||||
// against the current screenshot before clicking it
|
||||
checkForImageElementStaleness: true,
|
||||
|
||||
// whether before clicking on an image element Appium should re-determine the
|
||||
// position of the element on screen
|
||||
autoUpdateImageElementPosition: false,
|
||||
|
||||
// which method to use for tapping by coordinate for image elements. the
|
||||
// options are 'w3c' or 'mjsonwp'
|
||||
imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
|
||||
|
||||
// which method to use to save the matched image area in ImageElement class.
|
||||
// It is used for debugging purpose.
|
||||
getMatchedImageResult: false,
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import {node} from 'appium/support';
|
||||
import type {ImageSettings} from './types';
|
||||
|
||||
export const IMAGE_STRATEGY = '-image';
|
||||
|
||||
export const IMAGE_ELEMENT_PREFIX = 'appium-image-element-';
|
||||
export const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions';
|
||||
export const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions';
|
||||
export const IMAGE_TAP_STRATEGIES = [IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C] as const;
|
||||
export const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
|
||||
|
||||
export const MATCH_FEATURES_MODE = 'matchFeatures';
|
||||
export const GET_SIMILARITY_MODE = 'getSimilarity';
|
||||
export const MATCH_TEMPLATE_MODE = 'matchTemplate';
|
||||
|
||||
export const DEFAULT_MATCH_THRESHOLD = 0.4;
|
||||
|
||||
export const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
|
||||
|
||||
export const DEFAULT_SETTINGS = node.deepFreeze({
|
||||
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
|
||||
imageMatchMethod: '',
|
||||
fixImageFindScreenshotDims: true,
|
||||
fixImageTemplateSize: false,
|
||||
fixImageTemplateScale: false,
|
||||
defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
checkForImageElementStaleness: true,
|
||||
autoUpdateImageElementPosition: false,
|
||||
imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
|
||||
getMatchedImageResult: false,
|
||||
}) as ImageSettings;
|
||||
@@ -3,12 +3,23 @@ import {LRUCache} from 'lru-cache';
|
||||
import {errors} from 'appium/driver';
|
||||
import {ImageElement} from './image-element';
|
||||
import {compareImages} from './compare';
|
||||
import log from './logger';
|
||||
import {log} from './logger';
|
||||
import {
|
||||
DEFAULT_SETTINGS, MATCH_TEMPLATE_MODE, DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
DEFAULT_SETTINGS,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
||||
} from './constants';
|
||||
import sharp from 'sharp';
|
||||
import type {ExternalDriver, Element, Rect, Size} from '@appium/types';
|
||||
import type {
|
||||
ImageSettings,
|
||||
FindByImageOptions,
|
||||
Screenshot,
|
||||
ScreenshotScale,
|
||||
ImageTemplateSettings,
|
||||
OccurrenceResultWithVisualization,
|
||||
} from './types';
|
||||
|
||||
// Used to compare ratio and screen width
|
||||
// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
|
||||
@@ -19,28 +30,26 @@ const MAX_CACHE_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
/**
|
||||
* Checks if one rect fully contains another
|
||||
*
|
||||
* @param {import('@appium/types').Rect} templateRect The bounding rect
|
||||
* @param {import('@appium/types').Rect} rect The rect to be checked for containment
|
||||
* @returns {boolean} True if templateRect contains rect
|
||||
* @param templateRect The bounding rect
|
||||
* @param rect The rect to be checked for containment
|
||||
* @returns True if templateRect contains rect
|
||||
*/
|
||||
function containsRect(templateRect, rect) {
|
||||
return templateRect.x <= rect.x && templateRect.y <= rect.y
|
||||
&& rect.width <= templateRect.x + templateRect.width - rect.x
|
||||
&& rect.height <= templateRect.y + templateRect.height - rect.y;
|
||||
function containsRect(templateRect: Rect, rect: Rect): boolean {
|
||||
return (
|
||||
templateRect.x <= rect.x &&
|
||||
templateRect.y <= rect.y &&
|
||||
rect.width <= templateRect.x + templateRect.width - rect.x &&
|
||||
rect.height <= templateRect.y + templateRect.height - rect.y
|
||||
);
|
||||
}
|
||||
|
||||
const NO_OCCURRENCES_PATTERN = /Cannot find any occurrences/;
|
||||
const CONDITION_UNMET_PATTERN = /Condition unmet/;
|
||||
|
||||
export class ImageElementFinder {
|
||||
private _imgElCache: LRUCache<string, ImageElement>;
|
||||
|
||||
export default class ImageElementFinder {
|
||||
/** @type {LRUCache<string,ImageElement>} */
|
||||
_imgElCache;
|
||||
|
||||
/**
|
||||
* @param {number} max
|
||||
*/
|
||||
constructor(max = MAX_CACHE_ITEMS) {
|
||||
constructor(max: number = MAX_CACHE_ITEMS) {
|
||||
this._imgElCache = new LRUCache({
|
||||
ttl: MAX_CACHE_AGE_MS,
|
||||
updateAgeOnGet: true,
|
||||
@@ -48,57 +57,40 @@ export default class ImageElementFinder {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ImageElement} imgEl
|
||||
* @returns {Element}
|
||||
*/
|
||||
registerImageElement(imgEl) {
|
||||
registerImageElement(imgEl: ImageElement): Element {
|
||||
this._imgElCache.set(imgEl.id, imgEl);
|
||||
return imgEl.asElement();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} imgElId
|
||||
* @returns {ImageElement|undefined}
|
||||
*/
|
||||
getImageElement(imgElId) {
|
||||
getImageElement(imgElId: string): ImageElement | undefined {
|
||||
return this._imgElCache.get(imgElId);
|
||||
}
|
||||
|
||||
clearImageElements() {
|
||||
clearImageElements(): void {
|
||||
this._imgElCache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef FindByImageOptions
|
||||
* @property {boolean} [shouldCheckStaleness=false] - whether this call to find an
|
||||
* image is merely to check staleness. If so we can bypass a lot of logic
|
||||
* @property {boolean} [multiple=false] - Whether we are finding one element or
|
||||
* multiple
|
||||
* @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
|
||||
* ignore defaultImageTemplateScale. It can be used when you would like to
|
||||
* scale template with defaultImageTemplateScale setting.
|
||||
* @property {import('@appium/types').Rect?} [containerRect=null] - The bounding
|
||||
* rectangle to limit the search in
|
||||
*/
|
||||
|
||||
/**
|
||||
* Find a screen rect represented by an ImageElement corresponding to an image
|
||||
* template sent in by the client
|
||||
*
|
||||
* @param {Buffer} template - image used as a template to be
|
||||
* matched in the screenshot
|
||||
* @param {ExternalDriver} driver
|
||||
* @param {FindByImageOptions} opts - additional options
|
||||
* @param template - image used as a template to be matched in the screenshot
|
||||
* @param driver
|
||||
* @param opts - additional options
|
||||
*
|
||||
* @returns {Promise<Element|Element[]|ImageElement>} - WebDriver element with a special id prefix
|
||||
* @returns WebDriver element with a special id prefix
|
||||
*/
|
||||
async findByImage(
|
||||
template,
|
||||
driver,
|
||||
{shouldCheckStaleness = false, multiple = false, ignoreDefaultImageTemplateScale = false, containerRect = null}
|
||||
) {
|
||||
const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
template: Buffer,
|
||||
driver: ExternalDriver,
|
||||
{
|
||||
shouldCheckStaleness = false,
|
||||
multiple = false,
|
||||
ignoreDefaultImageTemplateScale = false,
|
||||
containerRect = null,
|
||||
}: FindByImageOptions = {}
|
||||
): Promise<Element | Element[] | ImageElement> {
|
||||
const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
const {
|
||||
imageMatchThreshold: threshold,
|
||||
imageMatchMethod,
|
||||
@@ -112,7 +104,7 @@ export default class ImageElementFinder {
|
||||
if (!driver.getWindowRect && !_.has(driver, 'getWindowSize')) {
|
||||
throw new Error("This driver does not support the required 'getWindowRect' command");
|
||||
}
|
||||
let screenSize;
|
||||
let screenSize: Size;
|
||||
if (driver.getWindowRect) {
|
||||
const screenRect = await driver.getWindowRect();
|
||||
screenSize = {
|
||||
@@ -120,7 +112,8 @@ export default class ImageElementFinder {
|
||||
height: screenRect.height,
|
||||
};
|
||||
} else {
|
||||
// @ts-ignore TODO: Drop the deprecated endpoint
|
||||
// TODO: Drop the deprecated endpoint
|
||||
// @ts-expect-error - deprecated getWindowSize method
|
||||
screenSize = await driver.getWindowSize();
|
||||
}
|
||||
|
||||
@@ -135,11 +128,10 @@ export default class ImageElementFinder {
|
||||
});
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const results: OccurrenceResultWithVisualization[] = [];
|
||||
let didFixTemplateImageScale = false;
|
||||
const performLookup = async () => {
|
||||
const performLookup = async (): Promise<boolean> => {
|
||||
try {
|
||||
|
||||
const {screenshot, scale} = await this.getScreenshotForImageFind(driver, screenSize);
|
||||
|
||||
if (!didFixTemplateImageScale) {
|
||||
@@ -154,7 +146,7 @@ export default class ImageElementFinder {
|
||||
didFixTemplateImageScale = true;
|
||||
}
|
||||
|
||||
const comparisonOpts = {
|
||||
const comparisonOpts: any = {
|
||||
threshold,
|
||||
visualize,
|
||||
multiple,
|
||||
@@ -163,23 +155,26 @@ export default class ImageElementFinder {
|
||||
comparisonOpts.method = imageMatchMethod;
|
||||
}
|
||||
|
||||
const pushIfOk = (el) => {
|
||||
if (containerRect && !containsRect(containerRect, el.rect)) {
|
||||
const pushIfOk = (el: any): boolean => {
|
||||
const result: OccurrenceResultWithVisualization = {
|
||||
rect: el.rect,
|
||||
score: el.score,
|
||||
visualization: el.visualization,
|
||||
};
|
||||
if (containerRect && !containsRect(containerRect, result.rect)) {
|
||||
log.debug(
|
||||
`The matched element rectangle ${JSON.stringify(el.rect)} is not located ` +
|
||||
`inside of the bounding rectangle ${JSON.stringify(containerRect)}, thus rejected`
|
||||
`The matched element rectangle ${JSON.stringify(result.rect)} is not located ` +
|
||||
`inside of the bounding rectangle ${JSON.stringify(containerRect)}, thus rejected`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
results.push(el);
|
||||
results.push(result);
|
||||
return true;
|
||||
};
|
||||
|
||||
const elOrEls = await compareImages(
|
||||
MATCH_TEMPLATE_MODE, screenshot, template, comparisonOpts
|
||||
);
|
||||
const elOrEls = await compareImages(MATCH_TEMPLATE_MODE, screenshot, template, comparisonOpts);
|
||||
return _.some((_.isArray(elOrEls) ? elOrEls : [elOrEls]).map(pushIfOk));
|
||||
} catch (err) {
|
||||
} catch (err: any) {
|
||||
// if compareImages fails, we'll get a specific error, but we should
|
||||
// retry, so trap that and just return false to trigger the next round of
|
||||
// implicitly waiting. For other errors, throw them to get out of the
|
||||
@@ -193,7 +188,7 @@ export default class ImageElementFinder {
|
||||
|
||||
try {
|
||||
await driver.implicitWaitForCondition(performLookup);
|
||||
} catch (err) {
|
||||
} catch (err: any) {
|
||||
// this `implicitWaitForCondition` method will throw a 'Condition unmet'
|
||||
// error if an element is not found eventually. In that case, we will
|
||||
// handle the element not found response below. In the case where get some
|
||||
@@ -239,12 +234,12 @@ export default class ImageElementFinder {
|
||||
/**
|
||||
* Ensure that the image template sent in for a find is of a suitable size
|
||||
*
|
||||
* @param {Buffer} template - template image
|
||||
* @param {import('@appium/types').Size} maxSize - size of the bounding rectangle
|
||||
* @param template - template image
|
||||
* @param maxSize - size of the bounding rectangle
|
||||
*
|
||||
* @returns {Promise<Buffer>} image, potentially resized
|
||||
* @returns image, potentially resized
|
||||
*/
|
||||
async ensureTemplateSize(template, maxSize) {
|
||||
async ensureTemplateSize(template: Buffer, maxSize: Size): Promise<Buffer> {
|
||||
const imgObj = sharp(template);
|
||||
const {width: tplWidth, height: tplHeight} = await imgObj.metadata();
|
||||
if (_.isNil(tplWidth) || _.isNil(tplHeight)) {
|
||||
@@ -261,32 +256,36 @@ export default class ImageElementFinder {
|
||||
|
||||
log.info(
|
||||
`Scaling template image from ${tplWidth}x${tplHeight} to match ` +
|
||||
`the bounding rectangle at ${maxSize.width}x${maxSize.height}`
|
||||
`the bounding rectangle at ${maxSize.width}x${maxSize.height}`
|
||||
);
|
||||
// otherwise, scale it to fit inside the bounding rectangle dimensions:
|
||||
// https://sharp.pixelplumbing.com/api-resize
|
||||
return await imgObj.resize({
|
||||
width: Math.trunc(maxSize.width),
|
||||
height: Math.trunc(maxSize.height),
|
||||
fit: 'inside',
|
||||
})
|
||||
.toBuffer();
|
||||
return await imgObj
|
||||
.resize({
|
||||
width: Math.trunc(maxSize.width),
|
||||
height: Math.trunc(maxSize.height),
|
||||
fit: 'inside',
|
||||
})
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the screenshot image that will be used for find by element, potentially
|
||||
* altering it in various ways based on user-requested settings
|
||||
*
|
||||
* @param {ExternalDriver} driver
|
||||
* @param {import('@appium/types').Size} screenSize - The original size of the screen
|
||||
* @param driver
|
||||
* @param screenSize - The original size of the screen
|
||||
*
|
||||
* @returns {Promise<Screenshot & {scale?: ScreenshotScale}>} PNG screenshot and ScreenshotScale
|
||||
* @returns PNG screenshot and ScreenshotScale
|
||||
*/
|
||||
async getScreenshotForImageFind(driver, screenSize) {
|
||||
async getScreenshotForImageFind(
|
||||
driver: ExternalDriver,
|
||||
screenSize: Size
|
||||
): Promise<Screenshot & {scale?: ScreenshotScale}> {
|
||||
if (!driver.getScreenshot) {
|
||||
throw new Error("This driver does not support the required 'getScreenshot' command");
|
||||
}
|
||||
const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
const {fixImageFindScreenshotDims} = settings;
|
||||
|
||||
const screenshot = Buffer.from(await driver.getScreenshot(), 'base64');
|
||||
@@ -301,7 +300,7 @@ export default class ImageElementFinder {
|
||||
if (screenSize.width < 1 || screenSize.height < 1) {
|
||||
log.warn(
|
||||
`The retrieved screen size ${screenSize.width}x${screenSize.height} does ` +
|
||||
`not seem to be valid. No changes will be applied to the screenshot`
|
||||
`not seem to be valid. No changes will be applied to the screenshot`
|
||||
);
|
||||
return {screenshot};
|
||||
}
|
||||
@@ -316,7 +315,7 @@ export default class ImageElementFinder {
|
||||
if (!shotWidth || shotWidth < 1 || !shotHeight || shotHeight < 1) {
|
||||
log.warn(
|
||||
`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +
|
||||
`not seem to be valid. No changes will be applied to the screenshot`
|
||||
`not seem to be valid. No changes will be applied to the screenshot`
|
||||
);
|
||||
return {screenshot};
|
||||
}
|
||||
@@ -334,21 +333,21 @@ export default class ImageElementFinder {
|
||||
// are two potential types of mismatch: aspect ratio mismatch and scale
|
||||
// mismatch. We need to detect and fix both
|
||||
|
||||
const scale = {xScale: 1.0, yScale: 1.0};
|
||||
const scale: ScreenshotScale = {xScale: 1.0, yScale: 1.0};
|
||||
|
||||
const screenAR = screenSize.width / screenSize.height;
|
||||
const shotAR = shotWidth / shotHeight;
|
||||
if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {
|
||||
log.info(
|
||||
`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +
|
||||
`screen aspect ratio '${screenAR}' (${screenSize.width}x${screenSize.height})`
|
||||
`screen aspect ratio '${screenAR}' (${screenSize.width}x${screenSize.height})`
|
||||
);
|
||||
} else {
|
||||
log.warn(
|
||||
`When trying to find an element, determined that the screen ` +
|
||||
`aspect ratio and screenshot aspect ratio are different. Screen ` +
|
||||
`is ${screenSize.width}x${screenSize.height} whereas screenshot is ` +
|
||||
`${shotWidth}x${shotHeight}.`
|
||||
`aspect ratio and screenshot aspect ratio are different. Screen ` +
|
||||
`is ${screenSize.width}x${screenSize.height} whereas screenshot is ` +
|
||||
`${shotWidth}x${shotHeight}.`
|
||||
);
|
||||
|
||||
// In the case where the x-scale and y-scale are different, we need to decide
|
||||
@@ -365,13 +364,12 @@ export default class ImageElementFinder {
|
||||
const xScale = (1.0 * shotWidth) / screenSize.width;
|
||||
const yScale = (1.0 * shotHeight) / screenSize.height;
|
||||
const scaleFactor = Math.min(xScale, yScale);
|
||||
const [newWidth, newHeight] = [shotWidth * scaleFactor, shotHeight * scaleFactor]
|
||||
.map(Math.trunc);
|
||||
const [newWidth, newHeight] = [shotWidth * scaleFactor, shotHeight * scaleFactor].map(Math.trunc);
|
||||
|
||||
log.warn(
|
||||
`Resizing screenshot to ${newWidth}x${newHeight} to match ` +
|
||||
`screen aspect ratio so that image element coordinates have a ` +
|
||||
`greater chance of being correct.`
|
||||
`screen aspect ratio so that image element coordinates have a ` +
|
||||
`greater chance of being correct.`
|
||||
);
|
||||
imgObj = imgObj.resize({
|
||||
width: newWidth,
|
||||
@@ -381,7 +379,8 @@ export default class ImageElementFinder {
|
||||
|
||||
scale.xScale *= scaleFactor;
|
||||
scale.yScale *= scaleFactor;
|
||||
[shotWidth, shotHeight] = [newWidth, newHeight];
|
||||
shotWidth = newWidth;
|
||||
shotHeight = newHeight;
|
||||
}
|
||||
|
||||
// Resize based on the screen dimensions only if both width and height are mismatched
|
||||
@@ -391,7 +390,7 @@ export default class ImageElementFinder {
|
||||
if (screenSize.width !== shotWidth && screenSize.height !== shotHeight) {
|
||||
log.info(
|
||||
`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
|
||||
`screen at ${screenSize.width}x${screenSize.height}`
|
||||
`screen at ${screenSize.width}x${screenSize.height}`
|
||||
);
|
||||
imgObj = imgObj.resize({
|
||||
width: Math.trunc(screenSize.width),
|
||||
@@ -409,40 +408,32 @@ export default class ImageElementFinder {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef ImageTemplateSettings
|
||||
* @property {boolean} [fixImageTemplateScale=false] - fixImageTemplateScale in device-settings
|
||||
* @property {number} [defaultImageTemplateScale=DEFAULT_TEMPLATE_IMAGE_SCALE] - defaultImageTemplateScale in device-settings
|
||||
* @property {boolean} [ignoreDefaultImageTemplateScale=false] - Ignore defaultImageTemplateScale if it has true.
|
||||
* If the template has been scaled to defaultImageTemplateScale or should ignore the scale,
|
||||
* this parameter should be true. e.g. click in image-element module
|
||||
* @property {number} [xScale=DEFAULT_FIX_IMAGE_TEMPLATE_SCALE] - Scale ratio for width
|
||||
* @property {number} [yScale=DEFAULT_FIX_IMAGE_TEMPLATE_SCALE] - Scale ratio for height
|
||||
|
||||
*/
|
||||
/**
|
||||
* Get a image that will be used for template matching.
|
||||
* Returns scaled image if scale ratio is provided.
|
||||
*
|
||||
* @param {Buffer} template - image used as a template to be
|
||||
* matched in the screenshot
|
||||
* @param {ImageTemplateSettings} opts - Image template scale related options
|
||||
* @param template - image used as a template to be matched in the screenshot
|
||||
* @param opts - Image template scale related options
|
||||
*
|
||||
* @returns {Promise<Buffer>} scaled template screenshot
|
||||
* @returns scaled template screenshot
|
||||
*/
|
||||
async fixImageTemplateScale(template, opts) {
|
||||
async fixImageTemplateScale(template: Buffer, opts?: ImageTemplateSettings): Promise<Buffer> {
|
||||
if (!opts) {
|
||||
return template;
|
||||
}
|
||||
|
||||
let {
|
||||
const {
|
||||
fixImageTemplateScale: fixTplScale = false,
|
||||
defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
defaultImageTemplateScale: initialDefaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
|
||||
ignoreDefaultImageTemplateScale = false,
|
||||
xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
||||
yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
||||
xScale: initialXScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
||||
yScale: initialYScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
||||
} = opts;
|
||||
|
||||
let defaultImageTemplateScale = initialDefaultImageTemplateScale;
|
||||
let xScale = initialXScale;
|
||||
let yScale = initialYScale;
|
||||
|
||||
if (ignoreDefaultImageTemplateScale) {
|
||||
defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;
|
||||
}
|
||||
@@ -467,12 +458,10 @@ export default class ImageElementFinder {
|
||||
|
||||
// Return if the scale is default, 1, value
|
||||
if (
|
||||
Math.round(xScale * FLOAT_PRECISION) ===
|
||||
Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION) &&
|
||||
Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION) &&
|
||||
Math.round(
|
||||
Number(
|
||||
yScale * FLOAT_PRECISION ===
|
||||
Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
|
||||
yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
|
||||
)
|
||||
)
|
||||
) {
|
||||
@@ -499,19 +488,3 @@ export default class ImageElementFinder {
|
||||
return await imgObj.toBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {import('@appium/types').ExternalDriver} ExternalDriver
|
||||
* @typedef {import('@appium/types').Element} Element
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef Screenshot
|
||||
* @property {Buffer} screenshot - screenshot image as PNG
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef ScreenshotScale
|
||||
* @property {number} xScale - Scale ratio for width
|
||||
* @property {number} yScale - Scale ratio for height
|
||||
*/
|
||||
+67
-85
@@ -1,46 +1,33 @@
|
||||
import _ from 'lodash';
|
||||
import {errors} from 'appium/driver';
|
||||
import {util} from '@appium/support';
|
||||
import log from './logger';
|
||||
import {log} from './logger';
|
||||
import {
|
||||
IMAGE_STRATEGY, DEFAULT_SETTINGS, IMAGE_TAP_STRATEGIES,
|
||||
IMAGE_ELEMENT_PREFIX, IMAGE_EL_TAP_STRATEGY_W3C,
|
||||
IMAGE_STRATEGY,
|
||||
DEFAULT_SETTINGS,
|
||||
IMAGE_TAP_STRATEGIES,
|
||||
IMAGE_ELEMENT_PREFIX,
|
||||
IMAGE_EL_TAP_STRATEGY_W3C,
|
||||
} from './constants';
|
||||
import type {Rect, Element, ExternalDriver, ActionSequence} from '@appium/types';
|
||||
import {ImageElementFinder} from './finder';
|
||||
import type {ImageSettings, Dimension, Position, ImageElementOpts} from './types';
|
||||
|
||||
const TAP_DURATION_MS = 125;
|
||||
|
||||
/**
|
||||
* @typedef Dimension
|
||||
* @property {number} width - width of rect
|
||||
* @property {number} height - height of rect
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef Position
|
||||
* @property {number} x - x coordinate
|
||||
* @property {number} y - y coordinate
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef ImageElementOpts
|
||||
* @property {Buffer} template - the image which was used to find this ImageElement
|
||||
* @property {Rect} rect - bounds of matched image element
|
||||
* @property {number} score The similarity score as a float number in range [0.0, 1.0].
|
||||
* 1.0 is the highest score (means both images are totally equal).
|
||||
* @property {Buffer?} match - the image which has matched marks. Defaults to null.
|
||||
* @property {import('./finder').default?} finder - the finder we can use to re-check stale elements
|
||||
* @property {import('@appium/types').Rect?} containerRect - The bounding
|
||||
* rectangle to limit the search in
|
||||
*/
|
||||
|
||||
/**
|
||||
* Representation of an "image element", which is simply a set of coordinates
|
||||
* and methods that can be used on that set of coordinates via the driver
|
||||
*/
|
||||
export default class ImageElement {
|
||||
/**
|
||||
* @param {ImageElementOpts} options
|
||||
*/
|
||||
export class ImageElement {
|
||||
readonly template: Buffer;
|
||||
rect: Rect;
|
||||
readonly id: string;
|
||||
readonly match: Buffer | null;
|
||||
readonly score: number;
|
||||
readonly finder: ImageElementFinder | null;
|
||||
readonly containerRect: Rect | null;
|
||||
|
||||
constructor({
|
||||
template,
|
||||
rect,
|
||||
@@ -48,7 +35,7 @@ export default class ImageElement {
|
||||
match = null,
|
||||
finder = null,
|
||||
containerRect = null,
|
||||
}) {
|
||||
}: ImageElementOpts) {
|
||||
this.template = template;
|
||||
this.rect = rect;
|
||||
this.id = `${IMAGE_ELEMENT_PREFIX}${util.uuidV4()}`;
|
||||
@@ -59,23 +46,23 @@ export default class ImageElement {
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Dimension} - dimension of element
|
||||
* @returns dimension of element
|
||||
*/
|
||||
get size() {
|
||||
get size(): Dimension {
|
||||
return {width: this.rect.width, height: this.rect.height};
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Position} - coordinates of top-left corner of element
|
||||
* @returns coordinates of top-left corner of element
|
||||
*/
|
||||
get location() {
|
||||
get location(): Position {
|
||||
return {x: this.rect.x, y: this.rect.y};
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Position} - coordinates of center of element
|
||||
* @returns coordinates of center of element
|
||||
*/
|
||||
get center() {
|
||||
get center(): Position {
|
||||
return {
|
||||
x: this.rect.x + this.rect.width / 2,
|
||||
y: this.rect.y + this.rect.height / 2,
|
||||
@@ -83,34 +70,33 @@ export default class ImageElement {
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string} - the base64-encoded original image used for matching
|
||||
* @returns the base64-encoded original image used for matching
|
||||
*/
|
||||
get originalImage() {
|
||||
get originalImage(): string {
|
||||
return this.template.toString('base64');
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string|null} - the base64-encoded image which has matched marks
|
||||
* @returns the base64-encoded image which has matched marks
|
||||
*/
|
||||
get matchedImage() {
|
||||
get matchedImage(): string | null {
|
||||
return this.match?.toString('base64') ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns {Element} - this image element as a WebElement
|
||||
* @returns this image element as a WebElement
|
||||
*/
|
||||
asElement() {
|
||||
asElement(): Element {
|
||||
return util.wrapElement(this.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ImageElement} other - an ImageElement to compare with this one
|
||||
* @param other - an ImageElement to compare with this one
|
||||
*
|
||||
* @returns {boolean} - whether the other element and this one have the same
|
||||
* properties
|
||||
* @returns whether the other element and this one have the same properties
|
||||
*/
|
||||
equals(other) {
|
||||
equals(other: ImageElement): boolean {
|
||||
return (
|
||||
this.rect.x === other.rect.x &&
|
||||
this.rect.y === other.rect.y &&
|
||||
@@ -123,12 +109,12 @@ export default class ImageElement {
|
||||
* Use a driver to tap the screen at the center of this ImageElement's
|
||||
* position
|
||||
*
|
||||
* @param {import('appium/driver').BaseDriver} driver - driver for calling actions with
|
||||
* @param driver - driver for calling actions with
|
||||
*/
|
||||
async click(driver) {
|
||||
async click(driver: ExternalDriver): Promise<void> {
|
||||
// before we click we need to make sure the element is actually still there
|
||||
// where we expect it to be
|
||||
const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
||||
const {
|
||||
autoUpdateImageElementPosition: updatePos,
|
||||
checkForImageElementStaleness,
|
||||
@@ -144,19 +130,20 @@ export default class ImageElement {
|
||||
);
|
||||
}
|
||||
|
||||
let newImgEl;
|
||||
let newImgEl: ImageElement | undefined;
|
||||
if (checkForImageElementStaleness || updatePos) {
|
||||
log.info('Checking image element for staleness before clicking');
|
||||
try {
|
||||
newImgEl = /** @type {ImageElement} */ (await (/** @type {import('./finder').default} */ (this.finder))
|
||||
.findByImage(
|
||||
this.template, driver, {
|
||||
shouldCheckStaleness: true,
|
||||
// Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
|
||||
// managed inside Appium after finidng image by template which managed by a user
|
||||
ignoreDefaultImageTemplateScale: true,
|
||||
containerRect: this.containerRect,
|
||||
}));
|
||||
if (!this.finder) {
|
||||
throw new errors.StaleElementReferenceError();
|
||||
}
|
||||
newImgEl = (await this.finder.findByImage(this.template, driver, {
|
||||
shouldCheckStaleness: true,
|
||||
// Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
|
||||
// managed inside Appium after finidng image by template which managed by a user
|
||||
ignoreDefaultImageTemplateScale: true,
|
||||
containerRect: this.containerRect,
|
||||
})) as ImageElement;
|
||||
} catch {
|
||||
throw new errors.StaleElementReferenceError();
|
||||
}
|
||||
@@ -188,7 +175,7 @@ export default class ImageElement {
|
||||
if (imageElementTapStrategy === IMAGE_EL_TAP_STRATEGY_W3C) {
|
||||
// set up a W3C action to click on the image by position
|
||||
log.info('Will tap using W3C actions');
|
||||
const action = {
|
||||
const action: ActionSequence = {
|
||||
type: 'pointer',
|
||||
id: 'mouse',
|
||||
parameters: {pointerType: 'touch'},
|
||||
@@ -231,34 +218,36 @@ export default class ImageElement {
|
||||
/**
|
||||
* Perform lookup of image element(s) inside of the current element
|
||||
*
|
||||
* @param {boolean} multiple - Whether to lookup multiple elements
|
||||
* @param {import('appium/driver').BaseDriver} driver - The driver to use for commands
|
||||
* @param {string[]} args = Rest of arguments for executeScripts
|
||||
* @returns {Promise<Element|Element[]|ImageElement>} - WebDriver element with a special id prefix
|
||||
* @param multiple - Whether to lookup multiple elements
|
||||
* @param driver - The driver to use for commands
|
||||
* @param args - Rest of arguments for executeScripts
|
||||
* @returns WebDriver element with a special id prefix
|
||||
*/
|
||||
async find(multiple, driver, ...args) {
|
||||
async find(multiple: boolean, driver: ExternalDriver, ...args: any[]): Promise<Element | Element[] | ImageElement> {
|
||||
const [strategy, selector] = args;
|
||||
if (strategy !== IMAGE_STRATEGY) {
|
||||
throw new errors.InvalidSelectorError(`Lookup strategies other than '${IMAGE_STRATEGY}' are not supported`);
|
||||
}
|
||||
return await (/** @type {import('./finder').default} */ (this.finder)).findByImage(
|
||||
Buffer.from(selector, 'base64'),
|
||||
driver,
|
||||
{multiple, containerRect: this.rect}
|
||||
);
|
||||
if (!this.finder) {
|
||||
throw new Error('Finder is not available');
|
||||
}
|
||||
return await this.finder.findByImage(Buffer.from(selector, 'base64'), driver, {
|
||||
multiple,
|
||||
containerRect: this.rect,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle various Appium commands that involve an image element
|
||||
*
|
||||
* @param {import('appium/driver').BaseDriver} driver - the driver to use for commands
|
||||
* @param {string} cmd - the name of the driver command
|
||||
* @param {any} imgEl - image element object
|
||||
* @param {string[]} args - Rest of arguments for executeScripts
|
||||
* @param driver - the driver to use for commands
|
||||
* @param cmd - the name of the driver command
|
||||
* @param imgEl - image element object
|
||||
* @param args - Rest of arguments for executeScripts
|
||||
*
|
||||
* @returns {Promise<any>} - the result of running a command
|
||||
* @returns the result of running a command
|
||||
*/
|
||||
static async execute(driver, imgEl, cmd, ...args) {
|
||||
static async execute(driver: ExternalDriver, imgEl: ImageElement, cmd: string, ...args: any[]): Promise<any> {
|
||||
switch (cmd) {
|
||||
case 'click':
|
||||
return await imgEl.click(driver);
|
||||
@@ -294,10 +283,3 @@ export default class ImageElement {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export {ImageElement};
|
||||
|
||||
/**
|
||||
* @typedef {import('@appium/types').Rect} Rect
|
||||
* @typedef {import('@appium/types').Element} Element
|
||||
*/
|
||||
@@ -0,0 +1,7 @@
|
||||
export {ImageElementPlugin, getImgElFromArgs} from './plugin';
|
||||
export {IMAGE_STRATEGY} from './constants';
|
||||
export {ImageElementFinder} from './finder';
|
||||
export {ImageElement} from './image-element';
|
||||
export * from './constants';
|
||||
export {compareImages} from './compare';
|
||||
export type * from './types';
|
||||
@@ -1,5 +0,0 @@
|
||||
import {logger} from 'appium/support';
|
||||
|
||||
const log = logger.getLogger('ImageElementPlugin');
|
||||
|
||||
export default log;
|
||||
@@ -0,0 +1,3 @@
|
||||
import {logger} from 'appium/support';
|
||||
|
||||
export const log = logger.getLogger('ImageElementPlugin');
|
||||
@@ -3,22 +3,26 @@ import {errors} from 'appium/driver';
|
||||
import {util} from '@appium/support';
|
||||
import {BasePlugin} from 'appium/plugin';
|
||||
import {compareImages} from './compare';
|
||||
import ImageElementFinder from './finder';
|
||||
import {ImageElementFinder} from './finder';
|
||||
import {ImageElement} from './image-element';
|
||||
import {IMAGE_STRATEGY, IMAGE_ELEMENT_PREFIX} from './constants';
|
||||
import type {ExternalDriver, Element, ActionSequence, MethodMap} from '@appium/types';
|
||||
import type {MatchingOptions, SimilarityOptions, OccurrenceOptions} from '@appium/opencv';
|
||||
|
||||
function getImgElFromArgs(args) {
|
||||
export function getImgElFromArgs(args: any[]): string | undefined {
|
||||
return args.find((arg) => _.isString(arg) && arg.startsWith(IMAGE_ELEMENT_PREFIX));
|
||||
}
|
||||
|
||||
export default class ImageElementPlugin extends BasePlugin {
|
||||
constructor(pluginName) {
|
||||
export class ImageElementPlugin extends BasePlugin {
|
||||
readonly finder: ImageElementFinder;
|
||||
|
||||
constructor(pluginName: string) {
|
||||
super(pluginName);
|
||||
this.finder = new ImageElementFinder();
|
||||
}
|
||||
|
||||
// this plugin supports a non-standard 'compare images' command
|
||||
static newMethodMap = /** @type {const} */ ({
|
||||
static newMethodMap: MethodMap<ImageElementPlugin> = {
|
||||
'/session/:sessionId/appium/compare_images': {
|
||||
POST: {
|
||||
command: 'compareImages',
|
||||
@@ -29,30 +33,33 @@ export default class ImageElementPlugin extends BasePlugin {
|
||||
neverProxy: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
} as const;
|
||||
|
||||
async compareImages(next, driver, ...args) {
|
||||
// @ts-ignore Arguments should be ok there
|
||||
return await compareImages(...args);
|
||||
async compareImages(
|
||||
next: () => Promise<any>,
|
||||
driver: ExternalDriver,
|
||||
mode: string,
|
||||
firstImage: string | Buffer,
|
||||
secondImage: string | Buffer,
|
||||
options?: MatchingOptions | SimilarityOptions | OccurrenceOptions
|
||||
): Promise<any> {
|
||||
return await compareImages(mode, firstImage, secondImage, options);
|
||||
}
|
||||
|
||||
async findElement(next, driver, ...args) {
|
||||
async findElement(next: () => Promise<any>, driver: ExternalDriver, ...args: any[]): Promise<any> {
|
||||
return await this._find(false, next, driver, ...args);
|
||||
}
|
||||
|
||||
async findElements(next, driver, ...args) {
|
||||
async findElements(next: () => Promise<any>, driver: ExternalDriver, ...args: any[]): Promise<any> {
|
||||
return await this._find(true, next, driver, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {boolean} multiple
|
||||
* @param {*} next
|
||||
* @param {*} driver
|
||||
* @param {...any} args
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
async _find(multiple, next, driver, ...args) {
|
||||
private async _find(
|
||||
multiple: boolean,
|
||||
next: () => Promise<any>,
|
||||
driver: ExternalDriver,
|
||||
...args: any[]
|
||||
): Promise<any> {
|
||||
const [strategy, selector] = args;
|
||||
|
||||
// if we're not actually finding by image, just do the normal thing
|
||||
@@ -63,7 +70,12 @@ export default class ImageElementPlugin extends BasePlugin {
|
||||
return await this.finder.findByImage(Buffer.from(selector, 'base64'), driver, {multiple});
|
||||
}
|
||||
|
||||
async handle(next, driver, cmdName, ...args) {
|
||||
async handle(
|
||||
next: () => Promise<any>,
|
||||
driver: ExternalDriver,
|
||||
cmdName: string,
|
||||
...args: any[]
|
||||
): Promise<any> {
|
||||
// if we have a command that involves an image element id, attempt to find the image element
|
||||
// and execute the command on it
|
||||
const imgElId = getImgElFromArgs(args);
|
||||
@@ -83,28 +95,22 @@ export default class ImageElementPlugin extends BasePlugin {
|
||||
return await next();
|
||||
}
|
||||
|
||||
async performActions(next, driver, ...args) {
|
||||
async performActions(
|
||||
next: () => Promise<any>,
|
||||
driver: ExternalDriver,
|
||||
actionSequences: ActionSequence[]
|
||||
): Promise<any> {
|
||||
// Replace with coordinates when ActionSequence includes image elements.
|
||||
const [actionSequences] = /** @type {[import('@appium/types').ActionSequence[]]} */ (args);
|
||||
for (const actionSequence of actionSequences) {
|
||||
for (const action of actionSequence.actions) {
|
||||
// The actions that can have an Element as the origin are "pointerMove" and "scroll".
|
||||
if (
|
||||
!_.isPlainObject(
|
||||
/** @type {{origin?: "viewport" | "pointer" | import('@appium/types').Element}} */ (
|
||||
action
|
||||
).origin,
|
||||
)
|
||||
) {
|
||||
if (!_.isPlainObject((action as any).origin)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const actionWithEl =
|
||||
/** @type {import('@appium/types').PointerMoveAction | import('@appium/types').ScrollAction} */ (
|
||||
action
|
||||
);
|
||||
const actionWithEl = action as any;
|
||||
|
||||
const elId = util.unwrapElement(/** @type {import('@appium/types').Element} */ (actionWithEl.origin));
|
||||
const elId = util.unwrapElement(actionWithEl.origin as Element);
|
||||
if (!_.startsWith(elId, IMAGE_ELEMENT_PREFIX)) {
|
||||
continue;
|
||||
}
|
||||
@@ -125,5 +131,3 @@ export default class ImageElementPlugin extends BasePlugin {
|
||||
return await next();
|
||||
}
|
||||
}
|
||||
|
||||
export {ImageElementPlugin, getImgElFromArgs, IMAGE_STRATEGY};
|
||||
@@ -0,0 +1,187 @@
|
||||
import type {Rect} from '@appium/types';
|
||||
import type {ImageElementFinder} from './finder';
|
||||
import type {MatchingResult, OccurrenceResult, SimilarityResult} from '@appium/opencv';
|
||||
import {
|
||||
IMAGE_EL_TAP_STRATEGY_W3C,
|
||||
IMAGE_EL_TAP_STRATEGY_MJSONWP,
|
||||
} from './constants';
|
||||
|
||||
/**
|
||||
* Image settings interface for device settings
|
||||
*/
|
||||
export interface ImageSettings {
|
||||
// value between 0 and 1 representing match strength, below which an image
|
||||
// element will not be found
|
||||
imageMatchThreshold: number;
|
||||
|
||||
// One of possible image matching methods.
|
||||
// Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
// for more details.
|
||||
// TM_CCOEFF_NORMED by default
|
||||
imageMatchMethod: string;
|
||||
|
||||
// if the image returned by getScreenshot differs in size or aspect ratio
|
||||
// from the screen, attempt to fix it automatically
|
||||
fixImageFindScreenshotDims: boolean;
|
||||
|
||||
// whether Appium should ensure that an image template sent in during image
|
||||
// element find should have its size adjusted so the match algorithm will not
|
||||
// complain
|
||||
fixImageTemplateSize: boolean;
|
||||
|
||||
// whether Appium should ensure that an image template sent in during image
|
||||
// element find should have its scale adjusted to display size so the match
|
||||
// algorithm will not complain.
|
||||
// e.g. iOS has `width=375, height=667` window rect, but its screenshot is
|
||||
// `width=750 × height=1334` pixels. This setting help to adjust the scale
|
||||
// if a user use `width=750 × height=1334` pixels's base template image.
|
||||
fixImageTemplateScale: boolean;
|
||||
|
||||
// Users might have scaled template image to reduce their storage size.
|
||||
// This setting allows users to scale a template image they send to Appium server
|
||||
// so that the Appium server compares the actual scale users originally had.
|
||||
// e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
|
||||
// the user can set {defaultImageTemplateScale: 4.0} to scale the small image
|
||||
// to the original one so that Appium can compare it as the original one.
|
||||
defaultImageTemplateScale: number;
|
||||
|
||||
// whether Appium should re-check that an image element can be matched
|
||||
// against the current screenshot before clicking it
|
||||
checkForImageElementStaleness: boolean;
|
||||
|
||||
// whether before clicking on an image element Appium should re-determine the
|
||||
// position of the element on screen
|
||||
autoUpdateImageElementPosition: boolean;
|
||||
|
||||
// which method to use for tapping by coordinate for image elements. the
|
||||
// options are 'w3c' or 'mjsonwp'
|
||||
imageElementTapStrategy: typeof IMAGE_EL_TAP_STRATEGY_W3C | typeof IMAGE_EL_TAP_STRATEGY_MJSONWP;
|
||||
|
||||
// which method to use to save the matched image area in ImageElement class.
|
||||
// It is used for debugging purpose.
|
||||
getMatchedImageResult: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for finding elements by image
|
||||
*/
|
||||
export interface FindByImageOptions {
|
||||
/** whether this call to find an image is merely to check staleness.
|
||||
* If so we can bypass a lot of logic */
|
||||
shouldCheckStaleness?: boolean;
|
||||
/** Whether we are finding one element or multiple */
|
||||
multiple?: boolean;
|
||||
/** Whether we ignore defaultImageTemplateScale. It can be used when you would like to
|
||||
* scale template with defaultImageTemplateScale setting. */
|
||||
ignoreDefaultImageTemplateScale?: boolean;
|
||||
/** The bounding rectangle to limit the search in */
|
||||
containerRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for creating an ImageElement
|
||||
*/
|
||||
export interface ImageElementOpts {
|
||||
/** the image which was used to find this ImageElement */
|
||||
template: Buffer;
|
||||
/** bounds of matched image element */
|
||||
rect: Rect;
|
||||
/** The similarity score as a float number in range [0.0, 1.0]. 1.0 is the highest
|
||||
* score (means both images are totally equal). */
|
||||
score: number;
|
||||
/** the image which has matched marks. Defaults to null. */
|
||||
match?: Buffer | null;
|
||||
/** the finder we can use to re-check stale elements */
|
||||
finder?: ImageElementFinder | null;
|
||||
/** The bounding rectangle to limit the search in */
|
||||
containerRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dimension interface for width and height
|
||||
*/
|
||||
export interface Dimension {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Position interface for x and y coordinates
|
||||
*/
|
||||
export interface Position {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Screenshot interface
|
||||
*/
|
||||
export interface Screenshot {
|
||||
screenshot: Buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Screenshot scale interface
|
||||
*/
|
||||
export interface ScreenshotScale {
|
||||
xScale: number;
|
||||
yScale: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Image template settings for scaling
|
||||
*/
|
||||
export interface ImageTemplateSettings {
|
||||
/** fixImageTemplateScale in device-settings */
|
||||
fixImageTemplateScale?: boolean;
|
||||
/** defaultImageTemplateScale in device-settings */
|
||||
defaultImageTemplateScale?: number;
|
||||
/** Ignore defaultImageTemplateScale if it has true. If the template
|
||||
* has been scaled to defaultImageTemplateScale or should ignore the scale,
|
||||
* this parameter should be true. e.g. click in image-element module */
|
||||
ignoreDefaultImageTemplateScale?: boolean;
|
||||
/** Scale ratio for width */
|
||||
xScale?: number;
|
||||
/** Scale ratio for height */
|
||||
yScale?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Occurrence result with visualization (string instead of Buffer)
|
||||
*/
|
||||
export interface OccurrenceResultWithVisualization {
|
||||
rect: Rect;
|
||||
score: number;
|
||||
visualization?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for results with string visualization instead of Buffer
|
||||
*/
|
||||
export interface Visualized {
|
||||
visualization?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Matching result with string visualization
|
||||
*/
|
||||
export type VisualizedMatchingResult = MatchingResult & Visualized;
|
||||
|
||||
/**
|
||||
* Occurrence result with string visualization
|
||||
*/
|
||||
export type VisualizedOccurrenceResult = OccurrenceResult & Visualized;
|
||||
|
||||
/**
|
||||
* Similarity result with string visualization
|
||||
*/
|
||||
export type VisualizedSimilarityResult = SimilarityResult & Visualized;
|
||||
|
||||
/**
|
||||
* Result type for image comparison operations
|
||||
*/
|
||||
export type ComparisonResult =
|
||||
| VisualizedMatchingResult
|
||||
| VisualizedOccurrenceResult
|
||||
| VisualizedSimilarityResult
|
||||
| VisualizedSimilarityResult[];
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/images-plugin",
|
||||
"version": "4.0.4",
|
||||
"version": "4.1.0",
|
||||
"description": "Plugin for working with images and image elements in Appium",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -23,26 +23,27 @@
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"author": "https://github.com/appium",
|
||||
"types": "./build/lib/plugin.d.ts",
|
||||
"main": "./build/lib/index.js",
|
||||
"types": "./build/lib/index.d.ts",
|
||||
"files": [
|
||||
"build",
|
||||
"docs",
|
||||
"lib",
|
||||
"index.js",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
"test:e2e": "mocha --exit --timeout 40s --slow 20s \"./test/e2e/**/*.spec.*js\"",
|
||||
"test:smoke": "node ./index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.*js\""
|
||||
"test:smoke": "node ./build/lib/index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.*ts\"",
|
||||
"test:e2e": "mocha --exit --timeout 40s --slow 20s \"./test/e2e/**/*.spec.*ts\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/opencv": "^4.0.3",
|
||||
"@appium/support": "^7.0.4",
|
||||
"lodash": "4.17.21",
|
||||
"lru-cache": "11.2.4",
|
||||
"@appium/opencv": "^4.1.0",
|
||||
"@appium/support": "^7.0.5",
|
||||
"lodash": "4.17.23",
|
||||
"lru-cache": "11.2.5",
|
||||
"sharp": "0.34.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
+27
-25
@@ -6,6 +6,10 @@ import {TEST_IMG_1_B64, TEST_IMG_2_B64, APPSTORE_IMG_PATH} from '../fixtures/ind
|
||||
import {pluginE2EHarness} from '@appium/plugin-test-support';
|
||||
import {tempDir, fs} from '@appium/support';
|
||||
import sharp from 'sharp';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
const THIS_PLUGIN_DIR = path.join(__dirname, '..', '..');
|
||||
const APPIUM_HOME = path.join(THIS_PLUGIN_DIR, 'local_appium_home');
|
||||
@@ -35,23 +39,9 @@ const WDIO_OPTS = {
|
||||
};
|
||||
|
||||
describe('ImageElementPlugin', function () {
|
||||
let server;
|
||||
let driver;
|
||||
|
||||
beforeEach(async function () {
|
||||
driver = await wdio(WDIO_OPTS);
|
||||
});
|
||||
|
||||
afterEach(async function () {
|
||||
if (driver) {
|
||||
await driver.deleteSession();
|
||||
}
|
||||
});
|
||||
|
||||
pluginE2EHarness({
|
||||
before,
|
||||
after,
|
||||
server,
|
||||
port: TEST_PORT,
|
||||
host: TEST_HOST,
|
||||
appiumHome: APPIUM_HOME,
|
||||
@@ -63,6 +53,18 @@ describe('ImageElementPlugin', function () {
|
||||
pluginSpec: THIS_PLUGIN_DIR,
|
||||
});
|
||||
|
||||
let driver: any;
|
||||
|
||||
beforeEach(async function () {
|
||||
driver = await wdio(WDIO_OPTS);
|
||||
});
|
||||
|
||||
afterEach(async function () {
|
||||
if (driver) {
|
||||
await driver.deleteSession();
|
||||
}
|
||||
});
|
||||
|
||||
it('should add the compareImages route', async function () {
|
||||
let comparison = await driver.compareImages(
|
||||
MATCH_FEATURES_MODE,
|
||||
@@ -70,24 +72,24 @@ describe('ImageElementPlugin', function () {
|
||||
TEST_IMG_2_B64,
|
||||
{}
|
||||
);
|
||||
comparison.count.should.eql(0);
|
||||
expect(comparison.count).to.eql(0);
|
||||
comparison = await driver.compareImages(
|
||||
GET_SIMILARITY_MODE,
|
||||
TEST_IMG_1_B64,
|
||||
TEST_IMG_2_B64,
|
||||
{}
|
||||
);
|
||||
comparison.score.should.be.above(0.2);
|
||||
expect(comparison.score).to.be.above(0.2);
|
||||
});
|
||||
|
||||
it('should find and interact with image elements', async function () {
|
||||
const imageEl = await driver.$(APPSTORE_IMG_PATH);
|
||||
const {x, y} = await imageEl.getLocation();
|
||||
const {width, height} = await imageEl.getSize();
|
||||
x.should.eql(28);
|
||||
y.should.eql(72);
|
||||
width.should.eql(80);
|
||||
height.should.eql(91);
|
||||
expect(x).to.eql(28);
|
||||
expect(y).to.eql(72);
|
||||
expect(width).to.eql(80);
|
||||
expect(height).to.eql(91);
|
||||
await imageEl.click();
|
||||
|
||||
const actionSequence = {
|
||||
@@ -115,15 +117,15 @@ describe('ImageElementPlugin', function () {
|
||||
await sharp(screenshotPath)
|
||||
.extract(
|
||||
{
|
||||
left: parseInt(width / 4, 10),
|
||||
top: parseInt(height / 4, 10),
|
||||
width: parseInt(width / 2, 10),
|
||||
height: parseInt(height / 2, 10),
|
||||
left: parseInt((width / 4).toString(), 10),
|
||||
top: parseInt((height / 4).toString(), 10),
|
||||
width: parseInt((width / 2).toString(), 10),
|
||||
height: parseInt((height / 2).toString(), 10),
|
||||
}
|
||||
)
|
||||
.toFile(tmpImgPath);
|
||||
const subEl = await imageEl.$(tmpImgPath);
|
||||
_.isNil(subEl).should.be.false;
|
||||
expect(_.isNil(subEl)).to.be.false;
|
||||
} finally {
|
||||
await fs.rimraf(tmpRoot);
|
||||
}
|
||||
+162
-160
@@ -3,40 +3,39 @@ import {util} from 'appium/support';
|
||||
import {BaseDriver} from 'appium/driver';
|
||||
import {ImageElementPlugin} from '../../lib/plugin';
|
||||
import {IMAGE_STRATEGY} from '../../lib/constants';
|
||||
import ImageElementFinder from '../../lib/finder';
|
||||
import {ImageElementFinder} from '../../lib/finder';
|
||||
import {ImageElement} from '../../lib/image-element';
|
||||
import sinon from 'sinon';
|
||||
import {createSandbox, type SinonSandbox} from 'sinon';
|
||||
import {TINY_PNG, TiNY_PNG_BUF, TINY_PNG_DIMS} from '../fixtures/index.cjs';
|
||||
import sharp from 'sharp';
|
||||
import * as compareModule from '../../lib/compare';
|
||||
import type {Constraints} from '@appium/types';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
const compareModule = require('../../lib/compare');
|
||||
use(chaiAsPromised);
|
||||
|
||||
const plugin = new ImageElementPlugin();
|
||||
const plugin = new ImageElementPlugin('test');
|
||||
|
||||
class PluginDriver extends BaseDriver {
|
||||
async getWindowRect() {}
|
||||
async getScreenshot() {}
|
||||
findElement(strategy, selector) {
|
||||
return plugin.findElement(_.noop, this, strategy, selector);
|
||||
class PluginDriver extends BaseDriver<Constraints> {
|
||||
constructor() {
|
||||
super({} as any);
|
||||
}
|
||||
findElements(strategy, selector) {
|
||||
return plugin.findElements(_.noop, this, strategy, selector);
|
||||
async getWindowRect(): Promise<any> {}
|
||||
async getScreenshot(): Promise<any> {}
|
||||
findElement(strategy: string, selector: string) {
|
||||
return plugin.findElement(async () => {}, this as any, strategy, selector);
|
||||
}
|
||||
findElements(strategy: string, selector: string) {
|
||||
return plugin.findElements(async () => {}, this as any, strategy, selector);
|
||||
}
|
||||
}
|
||||
|
||||
describe('finding elements by image', function () {
|
||||
let sandbox;
|
||||
let should;
|
||||
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
const chaiAsPromised = await import('chai-as-promised');
|
||||
chai.use(chaiAsPromised.default);
|
||||
should = chai.should();
|
||||
});
|
||||
let sandbox: SinonSandbox;
|
||||
|
||||
beforeEach(function () {
|
||||
sandbox = sinon.createSandbox();
|
||||
sandbox = createSandbox();
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
@@ -46,19 +45,19 @@ describe('finding elements by image', function () {
|
||||
describe('findElement', function () {
|
||||
it('should use a different special method to find element by image', async function () {
|
||||
const d = new PluginDriver();
|
||||
sandbox.stub(plugin.finder, 'findByImage').returns(true);
|
||||
sandbox.stub(d, 'findElOrElsWithProcessing').returns(false);
|
||||
await d.findElement(IMAGE_STRATEGY, 'foo').should.eventually.be.true;
|
||||
await d.findElements(IMAGE_STRATEGY, 'foo').should.eventually.be.true;
|
||||
sandbox.stub(plugin.finder, 'findByImage').returns(true as any);
|
||||
sandbox.stub(d, 'findElOrElsWithProcessing').returns(false as any);
|
||||
await expect(d.findElement(IMAGE_STRATEGY, 'foo')).to.eventually.be.true;
|
||||
await expect(d.findElements(IMAGE_STRATEGY, 'foo')).to.eventually.be.true;
|
||||
});
|
||||
it('should not be able to find image element from any other element', async function () {
|
||||
const d = new PluginDriver();
|
||||
await d
|
||||
.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')
|
||||
.should.be.rejectedWith(/Locator Strategy.+is not supported/);
|
||||
await d
|
||||
.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')
|
||||
.should.be.rejectedWith(/Locator Strategy.+is not supported/);
|
||||
await expect(
|
||||
d.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')
|
||||
).to.be.rejectedWith(/Locator Strategy.+is not supported/);
|
||||
await expect(
|
||||
d.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')
|
||||
).to.be.rejectedWith(/Locator Strategy.+is not supported/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -68,119 +67,121 @@ describe('finding elements by image', function () {
|
||||
const size = {width: 100, height: 200};
|
||||
const screenshot = Buffer.from('iVBORfoo', 'base64');
|
||||
const template = Buffer.from('iVBORbar', 'base64');
|
||||
/** @type {PluginDriver} */
|
||||
let d;
|
||||
/** @type {ImageElementFinder} */
|
||||
let f;
|
||||
/** @type {import('sinon').SinonStubbedMember<import('../../lib/compare').compareImages>} */
|
||||
let compareStub;
|
||||
let d: PluginDriver;
|
||||
let f: ImageElementFinder;
|
||||
let compareStub: sinon.SinonStub;
|
||||
|
||||
function basicStub(driver, finder) {
|
||||
function basicStub(driver: PluginDriver, finder: ImageElementFinder) {
|
||||
const rectStub = sandbox.stub(driver, 'getWindowRect').returns({
|
||||
x: 0,
|
||||
y: 0,
|
||||
...size,
|
||||
});
|
||||
} as any);
|
||||
const screenStub = sandbox.stub(finder, 'getScreenshotForImageFind')
|
||||
.returns({screenshot});
|
||||
.returns({screenshot} as any);
|
||||
return {rectStub, screenStub};
|
||||
}
|
||||
|
||||
function basicImgElVerify(imgElProto, finder) {
|
||||
function basicImgElVerify(imgElProto: any, finder: ImageElementFinder) {
|
||||
const imgElId = util.unwrapElement(imgElProto);
|
||||
const imgEl = finder.getImageElement(imgElId);
|
||||
(imgEl instanceof ImageElement).should.be.true;
|
||||
imgEl.rect.should.eql(rect);
|
||||
imgEl.score.should.eql(score);
|
||||
expect(imgEl).to.be.instanceOf(ImageElement);
|
||||
expect(imgEl!.rect).to.eql(rect);
|
||||
expect(imgEl!.score).to.eql(score);
|
||||
return imgEl;
|
||||
}
|
||||
|
||||
beforeEach(function () {
|
||||
d = new PluginDriver();
|
||||
f = new ImageElementFinder();
|
||||
compareStub = sandbox;
|
||||
compareStub = sandbox.stub(compareModule, 'compareImages');
|
||||
compareStub.resolves({rect, score});
|
||||
basicStub(d, f);
|
||||
});
|
||||
|
||||
it('should find an image element happypath', async function () {
|
||||
const imgElProto = await f.findByImage(template, d, {multiple: false});
|
||||
const imgElProto = await f.findByImage(template, d as any, {multiple: false});
|
||||
basicImgElVerify(imgElProto, f);
|
||||
});
|
||||
it('should find image elements happypath', async function () {
|
||||
compareStub.resolves([{rect, score}]);
|
||||
const els = await f.findByImage(template, d, {multiple: true});
|
||||
els.should.have.length(1);
|
||||
const els = await f.findByImage(template, d as any, {multiple: true});
|
||||
expect(els).to.have.length(1);
|
||||
basicImgElVerify(els[0], f);
|
||||
});
|
||||
it('should fail if driver does not support getWindowRect', async function () {
|
||||
d.getWindowRect = null;
|
||||
await f
|
||||
.findByImage(template, d, {multiple: false})
|
||||
.should.eventually.be.rejectedWith(/driver does not support/);
|
||||
(d as any).getWindowRect = null;
|
||||
await expect(
|
||||
f.findByImage(template, d as any, {multiple: false})
|
||||
).to.eventually.be.rejectedWith(/driver does not support/);
|
||||
});
|
||||
it('should fix template size if requested', async function () {
|
||||
const newTemplate = 'iVBORbaz';
|
||||
const newTemplateBuf = Buffer.from(newTemplate, 'base64');
|
||||
await d.settings.update({fixImageTemplateSize: true});
|
||||
sandbox.stub(f, 'ensureTemplateSize').returns(newTemplateBuf);
|
||||
const imgElProto = await f.findByImage(template, d, {multiple: false});
|
||||
await (d as any).settings.update({fixImageTemplateSize: true});
|
||||
sandbox.stub(f, 'ensureTemplateSize').resolves(newTemplateBuf);
|
||||
const imgElProto = await f.findByImage(template, d as any, {multiple: false});
|
||||
const imgEl = basicImgElVerify(imgElProto, f);
|
||||
imgEl.originalImage.should.eql(newTemplate);
|
||||
_.last(compareStub.args)[2].should.eql(newTemplateBuf);
|
||||
expect(imgEl!.originalImage).to.eql(newTemplate);
|
||||
expect(_.last(compareStub.args)![2]).to.eql(newTemplateBuf);
|
||||
});
|
||||
|
||||
it('should fix template size scale if requested', async function () {
|
||||
const newTemplate = 'iVBORbaz';
|
||||
const newTemplateBuf = Buffer.from(newTemplate, 'base64');
|
||||
await d.settings.update({fixImageTemplateScale: true});
|
||||
sandbox.stub(f, 'fixImageTemplateScale').returns(newTemplateBuf);
|
||||
const imgElProto = await f.findByImage(template, d, {multiple: false});
|
||||
await (d as any).settings.update({fixImageTemplateScale: true});
|
||||
sandbox.stub(f, 'fixImageTemplateScale').resolves(newTemplateBuf);
|
||||
const imgElProto = await f.findByImage(template, d as any, {multiple: false});
|
||||
const imgEl = basicImgElVerify(imgElProto, f);
|
||||
imgEl.originalImage.should.eql(newTemplate);
|
||||
_.last(compareStub.args)[2].should.eql(newTemplateBuf);
|
||||
expect(imgEl!.originalImage).to.eql(newTemplate);
|
||||
expect(_.last(compareStub.args)![2]).to.eql(newTemplateBuf);
|
||||
});
|
||||
it('should not fix template size scale if it is not requested', async function () {
|
||||
const newTemplate = 'iVBORbaz';
|
||||
const newTemplateBuf = Buffer.from(newTemplate, 'base64');
|
||||
await d.settings.update({});
|
||||
sandbox.stub(f, 'fixImageTemplateScale').returns(newTemplateBuf);
|
||||
f.fixImageTemplateScale.callCount.should.eql(0);
|
||||
await (d as any).settings.update({});
|
||||
// fixImageTemplateScale is always called, but should return the original template
|
||||
// when scaling is not requested. We verify this by checking the compareImages call
|
||||
// receives the original template, not a modified one.
|
||||
const imgElProto = await f.findByImage(template, d as any, {multiple: false});
|
||||
basicImgElVerify(imgElProto, f);
|
||||
// The template passed to compareImages should be the original (or same buffer reference)
|
||||
// when fixImageTemplateScale is not requested
|
||||
expect(compareStub.called).to.be.true;
|
||||
const lastCallArgs = _.last(compareStub.args);
|
||||
expect(lastCallArgs![2]).to.eql(template);
|
||||
});
|
||||
|
||||
it('should throw an error if template match fails', async function () {
|
||||
compareStub.rejects(new Error('Cannot find any occurrences'));
|
||||
await f
|
||||
.findByImage(template, d, {multiple: false})
|
||||
.should.be.rejectedWith(/element could not be located/);
|
||||
await expect(
|
||||
f.findByImage(template, d as any, {multiple: false})
|
||||
).to.be.rejectedWith(/element could not be located/);
|
||||
});
|
||||
it('should return empty array for multiple elements if template match fails', async function () {
|
||||
compareStub.rejects(new Error('Cannot find any occurrences'));
|
||||
await f.findByImage(template, d, {multiple: true}).should.eventually.eql([]);
|
||||
await expect(f.findByImage(template, d as any, {multiple: true})).to.eventually.eql([]);
|
||||
});
|
||||
it('should respect implicit wait', async function () {
|
||||
d.setImplicitWait(10);
|
||||
(d as any).setImplicitWait(10);
|
||||
compareStub.resetHistory();
|
||||
compareStub.returns({rect, score});
|
||||
compareStub.onFirstCall().throws(new Error('Cannot find any occurrences'));
|
||||
const imgElProto = await f.findByImage(template, d, {multiple: false});
|
||||
const imgElProto = await f.findByImage(template, d as any, {multiple: false});
|
||||
basicImgElVerify(imgElProto, f);
|
||||
compareStub.calledTwice.should.be.true;
|
||||
expect(compareStub.calledTwice).to.be.true;
|
||||
});
|
||||
it('should not add element to cache and return it directly when checking staleness', async function () {
|
||||
const imgEl = await f.findByImage(template, d, {
|
||||
const imgEl = await f.findByImage(template, d as any, {
|
||||
multiple: false,
|
||||
shouldCheckStaleness: true,
|
||||
});
|
||||
(imgEl instanceof ImageElement).should.be.true;
|
||||
_.isNil(f.getImageElement(imgEl.id)).should.be.true;
|
||||
imgEl.rect.should.eql(rect);
|
||||
}) as ImageElement;
|
||||
expect(imgEl).to.be.instanceOf(ImageElement);
|
||||
expect(_.isNil(f.getImageElement(imgEl.id))).to.be.true;
|
||||
expect(imgEl.rect).to.eql(rect);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fixImageTemplateScale', function () {
|
||||
let f;
|
||||
let f: ImageElementFinder;
|
||||
const basicTemplate = 'iVBORbaz';
|
||||
const basicTemplateBuf = Buffer.from(basicTemplate, 'base64');
|
||||
|
||||
@@ -190,90 +191,90 @@ describe('finding elements by image', function () {
|
||||
});
|
||||
|
||||
it('should not fix template size scale if no scale value', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(basicTemplateBuf, {fixImageTemplateScale: true})
|
||||
.should.eventually.eql(basicTemplateBuf);
|
||||
await expect(
|
||||
f.fixImageTemplateScale(basicTemplateBuf, {fixImageTemplateScale: true})
|
||||
).to.eventually.eql(basicTemplateBuf);
|
||||
});
|
||||
|
||||
it('should not fix template size scale if it is null', async function () {
|
||||
await f.fixImageTemplateScale(basicTemplateBuf, null).should.eventually.eql(basicTemplateBuf);
|
||||
await expect(f.fixImageTemplateScale(basicTemplateBuf, null as any)).to.eventually.eql(basicTemplateBuf);
|
||||
});
|
||||
|
||||
it('should not fix template size scale if it is not number', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(basicTemplateBuf, 'wrong-scale')
|
||||
.should.eventually.eql(basicTemplateBuf);
|
||||
await expect(
|
||||
f.fixImageTemplateScale(basicTemplateBuf, 'wrong-scale' as any)
|
||||
).to.eventually.eql(basicTemplateBuf);
|
||||
});
|
||||
|
||||
it('should fix template size scale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
fixImageTemplateScale: true,
|
||||
xScale: 1.5,
|
||||
yScale: 1.5,
|
||||
})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should not fix template size scale because of fixImageTemplateScale being false', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
fixImageTemplateScale: false,
|
||||
xScale: 1.5,
|
||||
yScale: 1.5,
|
||||
})
|
||||
.should.eventually.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should fix template size scale with default scale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
defaultImageTemplateScale: 4.0,
|
||||
})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should fix template size scale with default scale and image scale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
defaultImageTemplateScale: 4.0,
|
||||
fixImageTemplateScale: true,
|
||||
xScale: 1.5,
|
||||
yScale: 1.5,
|
||||
})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should not fix template size scale with default scale and image scale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
defaultImageTemplateScale: 4.0,
|
||||
fixImageTemplateScale: false,
|
||||
xScale: 1.5,
|
||||
yScale: 1.5,
|
||||
})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should not fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
defaultImageTemplateScale: 4.0,
|
||||
ignoreDefaultImageTemplateScale: true,
|
||||
})
|
||||
.should.eventually.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
|
||||
it('should ignore defaultImageTemplateScale to fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
|
||||
await f
|
||||
.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
await expect(
|
||||
f.fixImageTemplateScale(TiNY_PNG_BUF, {
|
||||
defaultImageTemplateScale: 4.0,
|
||||
ignoreDefaultImageTemplateScale: true,
|
||||
fixImageTemplateScale: true,
|
||||
xScale: 1.5,
|
||||
yScale: 1.5,
|
||||
})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -282,84 +283,85 @@ describe('finding elements by image', function () {
|
||||
|
||||
it('should not resize the template if it is smaller than the screen', async function () {
|
||||
const [width, height] = TINY_PNG_DIMS.map((n) => n * 2);
|
||||
await f.ensureTemplateSize(TiNY_PNG_BUF, {width, height}).should.eventually.eql(TiNY_PNG_BUF);
|
||||
await expect(f.ensureTemplateSize(TiNY_PNG_BUF, {width, height})).to.eventually.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
it('should not resize the template if it is the same size as the screen', async function () {
|
||||
const [width, height] = TINY_PNG_DIMS;
|
||||
await f.ensureTemplateSize(TiNY_PNG_BUF, {width, height}).should.eventually.eql(TiNY_PNG_BUF);
|
||||
await expect(f.ensureTemplateSize(TiNY_PNG_BUF, {width, height})).to.eventually.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
it('should resize the template if it is bigger than the screen', async function () {
|
||||
const [width, height] = TINY_PNG_DIMS.map((n) => n / 2);
|
||||
const newTemplateBuf = await f.ensureTemplateSize(TiNY_PNG_BUF, {width, height});
|
||||
newTemplateBuf.should.not.eql(TiNY_PNG_BUF);
|
||||
newTemplateBuf.length.should.be.below(TiNY_PNG_BUF.length);
|
||||
expect(newTemplateBuf).to.not.eql(TiNY_PNG_BUF);
|
||||
expect(newTemplateBuf.length).to.be.below(TiNY_PNG_BUF.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getScreenshotForImageFind', function () {
|
||||
let d;
|
||||
let f;
|
||||
let d: PluginDriver;
|
||||
let f: ImageElementFinder;
|
||||
|
||||
beforeEach(function () {
|
||||
d = new PluginDriver();
|
||||
f = new ImageElementFinder();
|
||||
sandbox.stub(d, 'getScreenshot').returns(TINY_PNG);
|
||||
sandbox.stub(d, 'getScreenshot').resolves(TINY_PNG);
|
||||
});
|
||||
|
||||
it('should fail if driver does not support getScreenshot', async function () {
|
||||
await new ImageElementFinder()
|
||||
.getScreenshotForImageFind(new BaseDriver())
|
||||
.should.eventually.be.rejectedWith(/driver does not support/);
|
||||
await expect(
|
||||
new ImageElementFinder()
|
||||
.getScreenshotForImageFind(new BaseDriver<Constraints>({} as any) as any, {width: 100, height: 100})
|
||||
).to.eventually.be.rejectedWith(/driver does not support/);
|
||||
});
|
||||
it('should not adjust or verify screenshot if asked not to by settings', async function () {
|
||||
await d.settings.update({fixImageFindScreenshotDims: false});
|
||||
await (d as any).settings.update({fixImageFindScreenshotDims: false});
|
||||
const [width, height] = TINY_PNG_DIMS.map((n) => n + 1);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
screenshot.should.eql(TiNY_PNG_BUF);
|
||||
should.equal(scale, undefined);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(screenshot).to.eql(TiNY_PNG_BUF);
|
||||
expect(scale).to.equal(undefined);
|
||||
});
|
||||
it('should return screenshot without adjustment if it matches screen size', async function () {
|
||||
const [width, height] = TINY_PNG_DIMS;
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
screenshot.should.eql(TiNY_PNG_BUF);
|
||||
should.equal(scale, undefined);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(screenshot).to.eql(TiNY_PNG_BUF);
|
||||
expect(scale).to.equal(undefined);
|
||||
});
|
||||
it('should return scaled screenshot with same aspect ratio if matching screen aspect ratio', async function () {
|
||||
const [width, height] = TINY_PNG_DIMS.map((n) => n * 1.5);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
screenshot.should.not.eql(TiNY_PNG_BUF);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(screenshot).to.not.eql(TiNY_PNG_BUF);
|
||||
const screenshotObj = sharp(screenshot);
|
||||
const {width: screenWidth, height: screenHeight} = await screenshotObj.metadata();
|
||||
screenWidth.should.eql(width);
|
||||
screenHeight.should.eql(height);
|
||||
scale.should.eql({xScale: 1.5, yScale: 1.5});
|
||||
expect(screenWidth).to.eql(width);
|
||||
expect(screenHeight).to.eql(height);
|
||||
expect(scale).to.eql({xScale: 1.5, yScale: 1.5});
|
||||
});
|
||||
it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio', async function () {
|
||||
// try first with portrait screen, screen = 8 x 12
|
||||
let [width, height] = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
|
||||
let expectedScale = {xScale: 2.67, yScale: 4};
|
||||
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
screenshot.should.not.eql(TiNY_PNG_BUF);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(screenshot).to.not.eql(TiNY_PNG_BUF);
|
||||
let screenshotObj = sharp(screenshot);
|
||||
let {width: screenWidth, height: screenHeight} = await screenshotObj.metadata();
|
||||
screenWidth.should.eql(width);
|
||||
screenHeight.should.eql(height);
|
||||
scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
|
||||
scale.yScale.should.eql(expectedScale.yScale);
|
||||
expect(screenWidth).to.eql(width);
|
||||
expect(screenHeight).to.eql(height);
|
||||
expect(scale!.xScale.toFixed(2)).to.eql(expectedScale.xScale.toString());
|
||||
expect(scale!.yScale).to.eql(expectedScale.yScale);
|
||||
|
||||
// then with landscape screen, screen = 12 x 8
|
||||
[width, height] = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
|
||||
expectedScale = {xScale: 4, yScale: 2.67};
|
||||
|
||||
const {screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
newScreen.should.not.eql(TiNY_PNG_BUF);
|
||||
const {screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(newScreen).to.not.eql(TiNY_PNG_BUF);
|
||||
screenshotObj = sharp(newScreen);
|
||||
({width: screenWidth, height: screenHeight} = await screenshotObj.metadata());
|
||||
screenWidth.should.eql(width);
|
||||
screenHeight.should.eql(height);
|
||||
newScale.xScale.should.eql(expectedScale.xScale);
|
||||
newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
|
||||
expect(screenWidth).to.eql(width);
|
||||
expect(screenHeight).to.eql(height);
|
||||
expect(newScale!.xScale).to.eql(expectedScale.xScale);
|
||||
expect(newScale!.yScale.toFixed(2)).to.eql(expectedScale.yScale.toString());
|
||||
});
|
||||
|
||||
it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio with fixImageTemplateScale', async function () {
|
||||
@@ -367,35 +369,35 @@ describe('finding elements by image', function () {
|
||||
let [width, height] = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
|
||||
let expectedScale = {xScale: 2.67, yScale: 4};
|
||||
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
screenshot.should.not.eql(TiNY_PNG_BUF);
|
||||
const {screenshot, scale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(screenshot).to.not.eql(TiNY_PNG_BUF);
|
||||
let screenshotObj = sharp(screenshot);
|
||||
let {width: screenWidth, height: screenHeight} = await screenshotObj.metadata();
|
||||
screenWidth.should.eql(width);
|
||||
screenHeight.should.eql(height);
|
||||
scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
|
||||
scale.yScale.should.eql(expectedScale.yScale);
|
||||
expect(screenWidth).to.eql(width);
|
||||
expect(screenHeight).to.eql(height);
|
||||
expect(scale!.xScale.toFixed(2)).to.eql(expectedScale.xScale.toString());
|
||||
expect(scale!.yScale).to.eql(expectedScale.yScale);
|
||||
// 8 x 12 stretched TINY_PNG
|
||||
await f
|
||||
.fixImageTemplateScale(screenshot, {fixImageTemplateScale: true, scale})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
await expect(
|
||||
f.fixImageTemplateScale(screenshot, {fixImageTemplateScale: true, xScale: scale!.xScale, yScale: scale!.yScale})
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
|
||||
// then with landscape screen, screen = 12 x 8
|
||||
[width, height] = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
|
||||
expectedScale = {xScale: 4, yScale: 2.67};
|
||||
|
||||
const {screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d, {width, height});
|
||||
newScreen.should.not.eql(TiNY_PNG_BUF);
|
||||
const {screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d as any, {width, height} as any);
|
||||
expect(newScreen).to.not.eql(TiNY_PNG_BUF);
|
||||
screenshotObj = sharp(newScreen);
|
||||
({width: screenWidth, height: screenHeight} = await screenshotObj.metadata());
|
||||
screenWidth.should.eql(width);
|
||||
screenHeight.should.eql(height);
|
||||
newScale.xScale.should.eql(expectedScale.xScale);
|
||||
newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
|
||||
expect(screenWidth).to.eql(width);
|
||||
expect(screenHeight).to.eql(height);
|
||||
expect(newScale!.xScale).to.eql(expectedScale.xScale);
|
||||
expect(newScale!.yScale.toFixed(2)).to.eql(expectedScale.yScale.toString());
|
||||
// 12 x 8 stretched TINY_PNG
|
||||
await f
|
||||
.fixImageTemplateScale(newScreen, {fixImageTemplateScale: true, scale})
|
||||
.should.eventually.not.eql(TiNY_PNG_BUF);
|
||||
await expect(
|
||||
f.fixImageTemplateScale(newScreen, {fixImageTemplateScale: true, xScale: newScale!.xScale, yScale: newScale!.yScale})
|
||||
).to.eventually.not.eql(TiNY_PNG_BUF);
|
||||
});
|
||||
});
|
||||
});
|
||||
+111
-88
@@ -1,29 +1,27 @@
|
||||
import _ from 'lodash';
|
||||
import {BaseDriver} from 'appium/driver';
|
||||
import {util} from 'appium/support';
|
||||
import ImageElementFinder from '../../lib/finder';
|
||||
import {ImageElementFinder} from '../../lib/finder';
|
||||
import {getImgElFromArgs} from '../../lib/plugin';
|
||||
import {ImageElement} from '../../lib/image-element';
|
||||
import sinon from 'sinon';
|
||||
import {createSandbox, type SinonSandbox} from 'sinon';
|
||||
import {IMAGE_ELEMENT_PREFIX} from '../../lib/constants';
|
||||
import type {Constraints} from '@appium/types';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
const defRect = {x: 100, y: 110, width: 50, height: 25};
|
||||
const defTemplate = Buffer.from('iVBORasdf', 'base64');
|
||||
|
||||
describe('ImageElement', function () {
|
||||
const driver = new BaseDriver();
|
||||
const driver = new BaseDriver<Constraints>({} as any);
|
||||
|
||||
let sandbox;
|
||||
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
const chaiAsPromised = await import('chai-as-promised');
|
||||
chai.use(chaiAsPromised.default);
|
||||
chai.should();
|
||||
});
|
||||
let sandbox: SinonSandbox;
|
||||
|
||||
beforeEach(function () {
|
||||
sandbox = sinon.createSandbox();
|
||||
sandbox = createSandbox();
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
@@ -35,8 +33,9 @@ describe('ImageElement', function () {
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
el.size.should.eql({width: defRect.width, height: defRect.height});
|
||||
expect(el.size).to.eql({width: defRect.width, height: defRect.height});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -45,8 +44,9 @@ describe('ImageElement', function () {
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
el.location.should.eql({x: defRect.x, y: defRect.y});
|
||||
expect(el.location).to.eql({x: defRect.x, y: defRect.y});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -55,8 +55,9 @@ describe('ImageElement', function () {
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
el.center.should.eql({
|
||||
expect(el.center).to.eql({
|
||||
x: defRect.x + defRect.width / 2,
|
||||
y: defRect.y + defRect.height / 2,
|
||||
});
|
||||
@@ -68,8 +69,9 @@ describe('ImageElement', function () {
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
util.unwrapElement(el.asElement()).should.match(/^appium-image-el/);
|
||||
expect(util.unwrapElement(el.asElement())).to.match(/^appium-image-el/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -77,46 +79,52 @@ describe('ImageElement', function () {
|
||||
it('should say two image elements with same rect are equal', function () {
|
||||
const el1 = new ImageElement({
|
||||
template: Buffer.from('foo'),
|
||||
rect: defRect
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
const el2 = new ImageElement({
|
||||
template: Buffer.from('bar'),
|
||||
rect: defRect
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
el1.equals(el2).should.be.true;
|
||||
el2.equals(el1).should.be.true;
|
||||
expect(el1.equals(el2)).to.be.true;
|
||||
expect(el2.equals(el1)).to.be.true;
|
||||
});
|
||||
it('should say two image elements with different rect are not equal', function () {
|
||||
const el1 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: {...defRect, x: 0}
|
||||
rect: {...defRect, x: 0},
|
||||
score: 1.0,
|
||||
});
|
||||
const el2 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
el1.equals(el2).should.be.false;
|
||||
el2.equals(el1).should.be.false;
|
||||
expect(el1.equals(el2)).to.be.false;
|
||||
expect(el2.equals(el1)).to.be.false;
|
||||
});
|
||||
});
|
||||
|
||||
describe('.click', function () {
|
||||
it('should reject an invalid tap strategy', async function () {
|
||||
const d = new BaseDriver();
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
await d.settings.update({imageElementTapStrategy: 'bad'});
|
||||
await el.click(d).should.be.rejectedWith(/Incorrect imageElementTapStrategy/);
|
||||
await expect(el.click(d as any)).to.be.rejectedWith(/Incorrect imageElementTapStrategy/);
|
||||
});
|
||||
it('should try to check for image element staleness, and throw if stale', async function () {
|
||||
const d = new BaseDriver();
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
const f = new ImageElementFinder();
|
||||
sandbox.stub(f, 'findByImage').throws();
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
finder: f,
|
||||
});
|
||||
// we need to check for staleness if explicitly requested to do so
|
||||
@@ -124,105 +132,111 @@ describe('ImageElement', function () {
|
||||
checkForImageElementStaleness: true,
|
||||
autoUpdateImageElementPosition: false,
|
||||
});
|
||||
await el.click(d).should.be.rejectedWith(/no longer attached/);
|
||||
await expect(el.click(d as any)).to.be.rejectedWith(/no longer attached/);
|
||||
|
||||
// and also if we are updating the element position
|
||||
await d.settings.update({
|
||||
checkForImageElementStaleness: false,
|
||||
autoUpdateImageElementPosition: true,
|
||||
});
|
||||
await el.click(d).should.be.rejectedWith(/no longer attached/);
|
||||
await expect(el.click(d as any)).to.be.rejectedWith(/no longer attached/);
|
||||
});
|
||||
it('should auto-update element position if requested', async function () {
|
||||
const d = new BaseDriver();
|
||||
d.performActions = _.noop;
|
||||
sandbox.stub(d, 'performActions');
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
(d as any).performActions = _.noop;
|
||||
sandbox.stub(d as any, 'performActions');
|
||||
const f = new ImageElementFinder();
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
finder: f,
|
||||
});
|
||||
const newRect = {...defRect, x: defRect.x + 10, y: defRect.y + 5};
|
||||
const elPos2 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: newRect,
|
||||
score: 1.0,
|
||||
finder: f,
|
||||
});
|
||||
sandbox.stub(f, 'findByImage').returns(elPos2);
|
||||
sandbox.stub(f, 'findByImage').returns(elPos2 as any);
|
||||
await d.settings.update({
|
||||
autoUpdateImageElementPosition: true,
|
||||
});
|
||||
el.rect.should.not.eql(newRect);
|
||||
await el.click(d);
|
||||
el.rect.should.eql(newRect);
|
||||
expect(el.rect).to.not.eql(newRect);
|
||||
await el.click(d as any);
|
||||
expect(el.rect).to.eql(newRect);
|
||||
});
|
||||
it('should tap the center of an element using w3c actions by default', async function () {
|
||||
const d = new BaseDriver();
|
||||
d.performActions = _.noop;
|
||||
const actionStub = sandbox.stub(d, 'performActions');
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
(d as any).performActions = _.noop;
|
||||
const actionStub = sandbox.stub(d as any, 'performActions');
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
// skip the staleness check for this test
|
||||
await d.settings.update({
|
||||
checkForImageElementStaleness: false,
|
||||
});
|
||||
await el.click(d);
|
||||
await el.click(d as any);
|
||||
const pointerMoveAction = actionStub.args[0][0][0].actions[0];
|
||||
pointerMoveAction.x.should.equal(el.center.x);
|
||||
pointerMoveAction.y.should.equal(el.center.y);
|
||||
expect(pointerMoveAction.x).to.equal(el.center.x);
|
||||
expect(pointerMoveAction.y).to.equal(el.center.y);
|
||||
});
|
||||
it('should fall back to touchactions if w3c actions do not exist on driver', async function () {
|
||||
const d = new BaseDriver();
|
||||
d.performTouch = _.noop;
|
||||
const actionStub = sandbox.stub(d, 'performTouch');
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
(d as any).performTouch = _.noop;
|
||||
const actionStub = sandbox.stub(d as any, 'performTouch');
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
// skip the staleness check for this test
|
||||
await d.settings.update({
|
||||
checkForImageElementStaleness: false,
|
||||
});
|
||||
await el.click(d);
|
||||
await el.click(d as any);
|
||||
const action = actionStub.args[0][0][0].options;
|
||||
action.x.should.equal(el.center.x);
|
||||
action.y.should.equal(el.center.y);
|
||||
expect(action.x).to.equal(el.center.x);
|
||||
expect(action.y).to.equal(el.center.y);
|
||||
});
|
||||
it('should use touchactions if requested', async function () {
|
||||
const d = new BaseDriver();
|
||||
d.performActions = _.noop;
|
||||
const w3cStub = sandbox.stub(d, 'performActions');
|
||||
d.performTouch = _.noop;
|
||||
const touchStub = sandbox.stub(d, 'performTouch');
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
(d as any).performActions = _.noop;
|
||||
const w3cStub = sandbox.stub(d as any, 'performActions');
|
||||
(d as any).performTouch = _.noop;
|
||||
const touchStub = sandbox.stub(d as any, 'performTouch');
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
// skip the staleness check for this test
|
||||
await d.settings.update({
|
||||
checkForImageElementStaleness: false,
|
||||
imageElementTapStrategy: 'touchActions',
|
||||
});
|
||||
await el.click(d);
|
||||
await el.click(d as any);
|
||||
const action = touchStub.args[0][0][0].options;
|
||||
action.x.should.equal(el.center.x);
|
||||
action.y.should.equal(el.center.y);
|
||||
w3cStub.callCount.should.eql(0);
|
||||
expect(action.x).to.equal(el.center.x);
|
||||
expect(action.y).to.equal(el.center.y);
|
||||
expect(w3cStub.callCount).to.eql(0);
|
||||
});
|
||||
it('should throw if driver does not implement any type of action', async function () {
|
||||
const d = new BaseDriver();
|
||||
const d = new BaseDriver<Constraints>({} as any);
|
||||
const el = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
// skip the staleness check for this test
|
||||
await d.settings.update({
|
||||
checkForImageElementStaleness: false,
|
||||
});
|
||||
await el.click(d).should.be.rejectedWith(/did not implement/);
|
||||
await expect(el.click(d as any)).to.be.rejectedWith(/did not implement/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -236,7 +250,7 @@ describe('ImageElement', function () {
|
||||
match: Buffer.from('aGFwcHkgdGVzdGluZw==', 'base64'),
|
||||
finder: f,
|
||||
});
|
||||
let clickStub;
|
||||
let clickStub: sinon.SinonStub;
|
||||
|
||||
before(function () {
|
||||
clickStub = sandbox.stub(imgEl, 'click');
|
||||
@@ -245,39 +259,39 @@ describe('ImageElement', function () {
|
||||
});
|
||||
|
||||
it('should reject executions for unsupported commands', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'foobar').should.be.rejectedWith(
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'foobar')).to.be.rejectedWith(
|
||||
/not yet been implemented/
|
||||
);
|
||||
});
|
||||
it('should get displayed status of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'elementDisplayed').should.eventually.be.true;
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'elementDisplayed')).to.eventually.be.true;
|
||||
});
|
||||
it('should get size of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getSize').should.eventually.eql({
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getSize')).to.eventually.eql({
|
||||
width: defRect.width,
|
||||
height: defRect.height,
|
||||
});
|
||||
});
|
||||
it('should get location of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getLocation').should.eventually.eql({
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getLocation')).to.eventually.eql({
|
||||
x: defRect.x,
|
||||
y: defRect.y,
|
||||
});
|
||||
});
|
||||
it('should get location in view of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getLocation').should.eventually.eql({
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getLocation')).to.eventually.eql({
|
||||
x: defRect.x,
|
||||
y: defRect.y,
|
||||
});
|
||||
});
|
||||
it('should get rect of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getElementRect').should.eventually.eql(defRect);
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getElementRect')).to.eventually.eql(defRect);
|
||||
});
|
||||
it('should get score of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getAttribute', 'score').should.eventually.eql(0);
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getAttribute', 'score')).to.eventually.eql(0);
|
||||
});
|
||||
it('should get visual of element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'getAttribute', 'visual').should.eventually.eql(
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'getAttribute', 'visual')).to.eventually.eql(
|
||||
'aGFwcHkgdGVzdGluZw=='
|
||||
);
|
||||
});
|
||||
@@ -285,24 +299,29 @@ describe('ImageElement', function () {
|
||||
const imgElement = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
await ImageElement.execute(
|
||||
driver,
|
||||
imgElement,
|
||||
'getAttribute',
|
||||
'visual'
|
||||
).should.eventually.eql(null);
|
||||
await expect(
|
||||
ImageElement.execute(
|
||||
driver as any,
|
||||
imgElement,
|
||||
'getAttribute',
|
||||
'visual'
|
||||
)
|
||||
).to.eventually.eql(null);
|
||||
});
|
||||
it('should not get other attribute', async function () {
|
||||
await ImageElement.execute(
|
||||
driver,
|
||||
imgEl,
|
||||
'getAttribute',
|
||||
'content-desc'
|
||||
).should.eventually.rejectedWith('Method has not yet been implemented');
|
||||
await expect(
|
||||
ImageElement.execute(
|
||||
driver as any,
|
||||
imgEl,
|
||||
'getAttribute',
|
||||
'content-desc'
|
||||
)
|
||||
).to.eventually.rejectedWith('Method has not yet been implemented');
|
||||
});
|
||||
it('should click element', async function () {
|
||||
await ImageElement.execute(driver, imgEl, 'click').should.eventually.be.true;
|
||||
await expect(ImageElement.execute(driver as any, imgEl, 'click')).to.eventually.be.true;
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -312,31 +331,35 @@ describe('image element LRU cache', function () {
|
||||
const el1 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
const el2 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
const finder = new ImageElementFinder();
|
||||
finder.registerImageElement(el1);
|
||||
el1.equals(finder.getImageElement(el1.id)).should.be.true;
|
||||
_.isUndefined(finder.getImageElement(el2.id)).should.be.true;
|
||||
expect(el1.equals(finder.getImageElement(el1.id)!)).to.be.true;
|
||||
expect(_.isUndefined(finder.getImageElement(el2.id))).to.be.true;
|
||||
});
|
||||
it('once cache reaches max size, should eject image elements', function () {
|
||||
const el1 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
const el2 = new ImageElement({
|
||||
template: defTemplate,
|
||||
rect: defRect,
|
||||
score: 1.0,
|
||||
});
|
||||
const finder = new ImageElementFinder(1);
|
||||
finder.registerImageElement(el1);
|
||||
_.isUndefined(finder.getImageElement(el1.id)).should.be.false;
|
||||
expect(_.isUndefined(finder.getImageElement(el1.id))).to.be.false;
|
||||
finder.registerImageElement(el2);
|
||||
_.isUndefined(finder.getImageElement(el1.id)).should.be.true;
|
||||
_.isUndefined(finder.getImageElement(el2.id)).should.be.false;
|
||||
expect(_.isUndefined(finder.getImageElement(el1.id))).to.be.true;
|
||||
expect(_.isUndefined(finder.getImageElement(el2.id))).to.be.false;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -344,15 +367,15 @@ describe('getImgElFromArgs', function () {
|
||||
it('should return the image element id from json obj in args', function () {
|
||||
const imgEl = `${IMAGE_ELEMENT_PREFIX}foo`;
|
||||
const args = [1, 'foo', imgEl];
|
||||
getImgElFromArgs(args).should.eql(imgEl);
|
||||
expect(getImgElFromArgs(args)).to.eql(imgEl);
|
||||
});
|
||||
it('should not return anything if image element id not in args', function () {
|
||||
const args = [1, 'foo'];
|
||||
_.isUndefined(getImgElFromArgs(args)).should.be.true;
|
||||
expect(_.isUndefined(getImgElFromArgs(args))).to.be.true;
|
||||
});
|
||||
it('should not find image element id in anything but prefix', function () {
|
||||
const notImgEl = `foo${IMAGE_ELEMENT_PREFIX}`;
|
||||
const args = [1, 'foo', notImgEl];
|
||||
_.isUndefined(getImgElFromArgs(args)).should.be.true;
|
||||
expect(_.isUndefined(getImgElFromArgs(args))).to.be.true;
|
||||
});
|
||||
});
|
||||
+80
-76
@@ -9,118 +9,122 @@ import {
|
||||
import {BaseDriver} from 'appium/driver';
|
||||
import {TEST_IMG_1_B64, TEST_IMG_2_B64, TEST_IMG_2_PART_B64} from '../fixtures/index.cjs';
|
||||
import {util} from '@appium/support';
|
||||
import type {ActionSequence, Constraints} from '@appium/types';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
describe('ImageElementPlugin#handle', function () {
|
||||
const next = () => {};
|
||||
const driver = new BaseDriver();
|
||||
const p = new ImageElementPlugin();
|
||||
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
const chaiAsPromised = await import('chai-as-promised');
|
||||
chai.use(chaiAsPromised.default);
|
||||
chai.should();
|
||||
});
|
||||
const next = async () => {};
|
||||
const driver = new BaseDriver<Constraints>({} as any);
|
||||
const p = new ImageElementPlugin('test');
|
||||
|
||||
describe('compareImages', function () {
|
||||
this.timeout(6000);
|
||||
it('should compare images via match features mode', async function () {
|
||||
const res = await p.compareImages(
|
||||
next,
|
||||
driver,
|
||||
driver as any,
|
||||
MATCH_FEATURES_MODE,
|
||||
TEST_IMG_1_B64,
|
||||
TEST_IMG_2_B64,
|
||||
{}
|
||||
);
|
||||
res.count.should.eql(0);
|
||||
expect(res).to.have.property('count');
|
||||
expect((res as any).count).to.eql(0);
|
||||
});
|
||||
it('should compare images via get similarity mode', async function () {
|
||||
const res = await p.compareImages(
|
||||
next,
|
||||
driver,
|
||||
driver as any,
|
||||
GET_SIMILARITY_MODE,
|
||||
Buffer.from(TEST_IMG_1_B64, 'base64'),
|
||||
Buffer.from(TEST_IMG_2_B64, 'base64'),
|
||||
{}
|
||||
);
|
||||
res.score.should.be.above(0.2);
|
||||
expect(res).to.have.property('score');
|
||||
expect((res as any).score).to.be.above(0.2);
|
||||
});
|
||||
it('should compare images via match template mode', async function () {
|
||||
const res = await p.compareImages(
|
||||
next,
|
||||
driver,
|
||||
driver as any,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
TEST_IMG_1_B64,
|
||||
TEST_IMG_2_B64,
|
||||
{}
|
||||
);
|
||||
res.rect.height.should.be.above(0);
|
||||
res.rect.width.should.be.above(0);
|
||||
res.score.should.be.above(0.2);
|
||||
expect(res).to.have.property('rect');
|
||||
expect((res as any).rect.height).to.be.above(0);
|
||||
expect((res as any).rect.width).to.be.above(0);
|
||||
expect((res as any).score).to.be.above(0.2);
|
||||
});
|
||||
it('should throw an error if comparison mode is not supported', async function () {
|
||||
await p
|
||||
.compareImages(next, driver, 'some mode', '', '')
|
||||
.should.eventually.be.rejectedWith(/comparison mode is unknown/);
|
||||
await expect(
|
||||
p.compareImages(next, driver as any, 'some mode', '', '')
|
||||
).to.eventually.be.rejectedWith(/comparison mode is unknown/);
|
||||
});
|
||||
it('should throw an error if image template is broken', async function () {
|
||||
await p.compareImages(
|
||||
next,
|
||||
driver,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
Buffer.from('d1423423424'),
|
||||
Buffer.from('d1423423424')
|
||||
).should.eventually.be.rejected;
|
||||
await expect(
|
||||
p.compareImages(
|
||||
next,
|
||||
driver as any,
|
||||
MATCH_TEMPLATE_MODE,
|
||||
Buffer.from('d1423423424'),
|
||||
Buffer.from('d1423423424')
|
||||
)
|
||||
).to.eventually.be.rejected;
|
||||
});
|
||||
it('should throw an error if image template is empty', async function () {
|
||||
await p.compareImages(next, driver, MATCH_TEMPLATE_MODE, Buffer.from(''), Buffer.from(''))
|
||||
.should.eventually.be.rejected;
|
||||
await expect(
|
||||
p.compareImages(next, driver as any, MATCH_TEMPLATE_MODE, Buffer.from(''), Buffer.from(''))
|
||||
).to.eventually.be.rejected;
|
||||
});
|
||||
});
|
||||
|
||||
describe('findElement(s)', function () {
|
||||
driver.settings = {getSettings: () => ({})};
|
||||
driver.isW3CProtocol = () => true;
|
||||
driver.getScreenshot = () => TEST_IMG_2_B64;
|
||||
driver.getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
|
||||
(driver as any).settings = {getSettings: () => ({})};
|
||||
(driver as any).isW3CProtocol = () => true;
|
||||
(driver as any).getScreenshot = () => TEST_IMG_2_B64;
|
||||
(driver as any).getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
|
||||
it('should defer execution to regular command if not a find command', async function () {
|
||||
const next = () => true;
|
||||
await p.handle(next, driver, 'sendKeys').should.eventually.become(true);
|
||||
const next = async () => true;
|
||||
await expect(p.handle(next, driver as any, 'sendKeys')).to.eventually.become(true);
|
||||
});
|
||||
it('should defer execution to regular command if it is a find command but a different strategy', async function () {
|
||||
const next = () => true;
|
||||
await p.findElement(next, driver, 'xpath', '//foo/bar').should.eventually.become(true);
|
||||
await p.findElements(next, driver, 'xpath', '//foo/bar').should.eventually.become(true);
|
||||
const next = async () => true;
|
||||
await expect(p.findElement(next, driver as any, 'xpath', '//foo/bar')).to.eventually.become(true);
|
||||
await expect(p.findElements(next, driver as any, 'xpath', '//foo/bar')).to.eventually.become(true);
|
||||
});
|
||||
it('should find an image element inside a screenshot', async function () {
|
||||
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
util.unwrapElement(el).should.include('appium-image-element');
|
||||
const el = await p.findElement(next, driver as any, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
expect(util.unwrapElement(el)).to.include('appium-image-element');
|
||||
});
|
||||
it('should find image elements inside a screenshot', async function () {
|
||||
const els = await p.findElements(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
els.should.have.length(1);
|
||||
util.unwrapElement(els[0]).should.include('appium-image-element');
|
||||
const els = await p.findElements(next, driver as any, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
expect(els).to.have.length(1);
|
||||
expect(util.unwrapElement(els[0])).to.include('appium-image-element');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Element interactions', function () {
|
||||
let elId;
|
||||
let elId: string;
|
||||
before(async function () {
|
||||
driver.settings = {getSettings: () => ({})};
|
||||
driver.isW3CProtocol = () => true;
|
||||
driver.getScreenshot = () => TEST_IMG_2_B64;
|
||||
driver.getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
|
||||
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
(driver as any).settings = {getSettings: () => ({})};
|
||||
(driver as any).isW3CProtocol = () => true;
|
||||
(driver as any).getScreenshot = () => TEST_IMG_2_B64;
|
||||
(driver as any).getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
|
||||
const el = await p.findElement(next, driver as any, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
elId = util.unwrapElement(el);
|
||||
});
|
||||
it('should click on the screen coords of the middle of the element', async function () {
|
||||
let action = null;
|
||||
driver.performActions = (a) => {
|
||||
let action: ActionSequence[] | null = null;
|
||||
(driver as any).performActions = async (a: ActionSequence[]) => {
|
||||
action = a;
|
||||
};
|
||||
await p.handle(next, driver, 'click', elId);
|
||||
action.should.eql([
|
||||
await p.handle(next, driver as any, 'click', elId);
|
||||
expect(action).to.eql([
|
||||
{
|
||||
type: 'pointer',
|
||||
id: 'mouse',
|
||||
@@ -135,22 +139,22 @@ describe('ImageElementPlugin#handle', function () {
|
||||
]);
|
||||
});
|
||||
it('should always say the element is displayed', async function () {
|
||||
await p.handle(next, driver, 'elementDisplayed', elId).should.eventually.be.true;
|
||||
await expect(p.handle(next, driver as any, 'elementDisplayed', elId)).to.eventually.be.true;
|
||||
});
|
||||
it('should return the matched region size', async function () {
|
||||
await p.handle(next, driver, 'getSize', elId).should.eventually.eql({
|
||||
await expect(p.handle(next, driver as any, 'getSize', elId)).to.eventually.eql({
|
||||
width: 48,
|
||||
height: 48,
|
||||
});
|
||||
});
|
||||
it('should return the matched region location', async function () {
|
||||
await p.handle(next, driver, 'getLocation', elId).should.eventually.eql({
|
||||
await expect(p.handle(next, driver as any, 'getLocation', elId)).to.eventually.eql({
|
||||
x: 0,
|
||||
y: 16,
|
||||
});
|
||||
});
|
||||
it('should return the region rect', async function () {
|
||||
await p.handle(next, driver, 'getElementRect', elId).should.eventually.eql({
|
||||
await expect(p.handle(next, driver as any, 'getElementRect', elId)).to.eventually.eql({
|
||||
x: 0,
|
||||
y: 16,
|
||||
height: 48,
|
||||
@@ -158,36 +162,36 @@ describe('ImageElementPlugin#handle', function () {
|
||||
});
|
||||
});
|
||||
it('should return the match score as the score attr', async function () {
|
||||
await p.handle(next, driver, 'getAttribute', 'score', elId).should.eventually.be.above(0.7);
|
||||
await expect(p.handle(next, driver as any, 'getAttribute', 'score', elId)).to.eventually.be.above(0.7);
|
||||
});
|
||||
it('should return the match visualization as the visual attr', async function () {
|
||||
driver.settings = {
|
||||
(driver as any).settings = {
|
||||
getSettings: () => ({
|
||||
getMatchedImageResult: true,
|
||||
}),
|
||||
};
|
||||
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
const el = await p.findElement(next, driver as any, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
elId = util.unwrapElement(el);
|
||||
await p
|
||||
.handle(next, driver, 'getAttribute', 'visual', elId)
|
||||
.should.eventually.include('iVBOR');
|
||||
await expect(
|
||||
p.handle(next, driver as any, 'getAttribute', 'visual', elId)
|
||||
).to.eventually.include('iVBOR');
|
||||
});
|
||||
it('should not allow any other attrs', async function () {
|
||||
await p
|
||||
.handle(next, driver, 'getAttribute', 'rando', elId)
|
||||
.should.eventually.be.rejectedWith(/not yet/i);
|
||||
await expect(
|
||||
p.handle(next, driver as any, 'getAttribute', 'rando', elId)
|
||||
).to.eventually.be.rejectedWith(/not yet/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('performActions', function () {
|
||||
let imageEl;
|
||||
let nativeEl;
|
||||
let imageEl: any;
|
||||
let nativeEl: any;
|
||||
before(async function () {
|
||||
imageEl = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
imageEl = await p.findElement(next, driver as any, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
|
||||
nativeEl = util.wrapElement('dummy-native-element-id');
|
||||
});
|
||||
it('should replace with coords of the image elements in pointerMove, scroll actions', async function () {
|
||||
const actionSequences = [
|
||||
const actionSequences: ActionSequence[] = [
|
||||
{
|
||||
type: 'pointer',
|
||||
id: 'mouse',
|
||||
@@ -205,8 +209,8 @@ describe('ImageElementPlugin#handle', function () {
|
||||
],
|
||||
},
|
||||
];
|
||||
await p.performActions(next, driver, actionSequences);
|
||||
actionSequences.should.eql([
|
||||
await p.performActions(next, driver as any, actionSequences);
|
||||
expect(actionSequences).to.eql([
|
||||
{
|
||||
type: 'pointer',
|
||||
id: 'mouse',
|
||||
@@ -226,7 +230,7 @@ describe('ImageElementPlugin#handle', function () {
|
||||
]);
|
||||
});
|
||||
it('should not be modified except pointerMove and scroll actions includes image element as origin', async function () {
|
||||
const actionSequences = [
|
||||
const actionSequences: ActionSequence[] = [
|
||||
{
|
||||
type: 'pointer',
|
||||
id: 'mouse',
|
||||
@@ -261,8 +265,8 @@ describe('ImageElementPlugin#handle', function () {
|
||||
},
|
||||
];
|
||||
const clone = _.cloneDeep(actionSequences);
|
||||
await p.performActions(next, driver, actionSequences);
|
||||
actionSequences.should.eql(clone);
|
||||
await p.performActions(next, driver as any, actionSequences);
|
||||
expect(actionSequences).to.eql(clone);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -6,10 +6,11 @@
|
||||
"strict": false,
|
||||
"paths": {
|
||||
"@appium/opencv": ["../opencv"]
|
||||
}
|
||||
},
|
||||
"types": ["mocha", "chai", "sinon", "chai-as-promised"]
|
||||
},
|
||||
"extends": "@appium/tsconfig/tsconfig.plugin.json",
|
||||
"include": ["lib"],
|
||||
"include": ["lib", "test"],
|
||||
"references": [
|
||||
{
|
||||
"path": "../opencv"
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [2.0.4](https://github.com/appium/appium/compare/@appium/logger@2.0.3...@appium/logger@2.0.4) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/logger
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [2.0.3](https://github.com/appium/appium/compare/@appium/logger@2.0.2...@appium/logger@2.0.3) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/logger
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/logger",
|
||||
"version": "2.0.3",
|
||||
"version": "2.0.4",
|
||||
"author": "https://github.com/appium",
|
||||
"description": "A Universal Logger For The Appium Ecosystem",
|
||||
"repository": {
|
||||
@@ -16,6 +16,7 @@
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test",
|
||||
"CHANGELOG.md"
|
||||
],
|
||||
"directories": {
|
||||
@@ -28,8 +29,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"console-control-strings": "1.1.0",
|
||||
"lodash": "4.17.21",
|
||||
"lru-cache": "11.2.4",
|
||||
"lodash": "4.17.23",
|
||||
"lru-cache": "11.2.5",
|
||||
"set-blocking": "2.0.0"
|
||||
},
|
||||
"license": "ISC",
|
||||
|
||||
@@ -3,6 +3,20 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [4.1.0](https://github.com/appium/appium/compare/@appium/opencv@4.0.3...@appium/opencv@4.1.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **opencv:** Migrate to typescript ([#21881](https://github.com/appium/appium/issues/21881)) ([fec5f3a](https://github.com/appium/appium/commit/fec5f3a377de4804920f5e8293ae5f198a7d4473))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [4.0.3](https://github.com/appium/appium/compare/@appium/opencv@4.0.2...@appium/opencv@4.0.3) (2025-11-12)
|
||||
|
||||
**Note:** Version bump only for package @appium/opencv
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
module.exports = require('./build/lib');
|
||||
@@ -1,30 +1,29 @@
|
||||
interface Deletable {
|
||||
delete(): void;
|
||||
}
|
||||
|
||||
export class OpenCvAutoreleasePool {
|
||||
/**
|
||||
* @type {Set}
|
||||
*/
|
||||
_items;
|
||||
private readonly _items: Set<Deletable>;
|
||||
|
||||
constructor() {
|
||||
this._items = new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {...T} items
|
||||
* @return {T[]|T} the same items
|
||||
*/
|
||||
add(...items) {
|
||||
add<T extends Deletable>(item: T): T;
|
||||
add<T extends Deletable>(...items: T[]): T | T[] {
|
||||
for (const item of items) {
|
||||
this._items.add(item);
|
||||
}
|
||||
return items.length === 1 ? items[0] : items;
|
||||
}
|
||||
|
||||
drain() {
|
||||
drain(): void {
|
||||
for (const item of this._items) {
|
||||
try {
|
||||
item.delete();
|
||||
} catch {}
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
this._items.clear();
|
||||
}
|
||||
@@ -1,685 +0,0 @@
|
||||
import _ from 'lodash';
|
||||
import {Buffer} from 'node:buffer';
|
||||
import B from 'bluebird';
|
||||
import sharp from 'sharp';
|
||||
import { OpenCvAutoreleasePool } from './autorelease-pool';
|
||||
|
||||
/** @type {any} */
|
||||
let cv;
|
||||
|
||||
/**
|
||||
* @typedef Region
|
||||
* @property {number} left - The offset from the left side
|
||||
* @property {number} top - The offset from the top
|
||||
* @property {number} width - The width
|
||||
* @property {number} height - The height
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef Point
|
||||
* @property {number} x - The x coordinate
|
||||
* @property {number} y - The y coordinate
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef Rect
|
||||
* @property {number} x - The top left coordinate
|
||||
* @property {number} y - The bottom right coordinate
|
||||
* @property {number} width - The width
|
||||
* @property {number} height - The height
|
||||
*/
|
||||
|
||||
const DEFAULT_MATCH_THRESHOLD = 0.5;
|
||||
const MATCH_NEIGHBOUR_THRESHOLD = 10;
|
||||
|
||||
const AVAILABLE_DETECTORS = {
|
||||
AKAZE: 'AKAZE',
|
||||
AGAST: 'AgastFeatureDetector',
|
||||
BRISK: 'BRISK',
|
||||
FAST: 'FastFeatureDetector',
|
||||
GFTT: 'GFTTDetector',
|
||||
KAZE: 'KAZE',
|
||||
MSER: 'MSER',
|
||||
ORB: 'ORB',
|
||||
};
|
||||
|
||||
const AVAILABLE_MATCHING_FUNCTIONS = {
|
||||
FlannBased: 'FlannBased',
|
||||
BruteForce: 'BruteForce',
|
||||
BruteForceL1: 'BruteForce-L1',
|
||||
BruteForceHamming: 'BruteForce-Hamming',
|
||||
BruteForceHammingLut: 'BruteForce-HammingLUT',
|
||||
BruteForceSL2: 'BruteForce-SL2',
|
||||
};
|
||||
|
||||
const MATCHING_METHODS = [
|
||||
'TM_CCOEFF',
|
||||
'TM_CCOEFF_NORMED',
|
||||
'TM_CCORR',
|
||||
'TM_CCORR_NORMED',
|
||||
'TM_SQDIFF',
|
||||
'TM_SQDIFF_NORMED',
|
||||
];
|
||||
const DEFAULT_MATCHING_METHOD = 'TM_CCOEFF_NORMED';
|
||||
|
||||
/**
|
||||
* Transforms matching method name to the actual
|
||||
* constant value from OpenCV library
|
||||
*
|
||||
* @param {string} name One of supported method names
|
||||
* (see MATCHING_METHODS array above)
|
||||
* @returns {number} The method value
|
||||
* @throws {Error} if an unsupported method name is given
|
||||
*/
|
||||
function toMatchingMethod(name) {
|
||||
if (!MATCHING_METHODS.includes(name)) {
|
||||
throw new Error(
|
||||
`The matching method '${name}' is unknown. ` +
|
||||
`Only the following matching methods are supported: ${MATCHING_METHODS}`
|
||||
);
|
||||
}
|
||||
return cv[name];
|
||||
}
|
||||
|
||||
/**
|
||||
* Spins until the opencv-bindings module is fully loaded
|
||||
*/
|
||||
async function initOpenCv() {
|
||||
cv = require('opencv-bindings');
|
||||
while (!cv.getBuildInformation) {
|
||||
await B.delay(500);
|
||||
}
|
||||
// opencv-bindings sets a global unhandledRejection handler of an abort, which we don't want, so
|
||||
// undo it here. https://github.com/opencv/opencv/issues/21481
|
||||
process.removeAllListeners('unhandledRejection');
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef MatchComputationResult
|
||||
* @property {OpenCVBindings['Mat']} descriptor - OpenCV match descriptor
|
||||
* @property {OpenCVBindings['KeyPointVector']} keyPoints - The array of key points
|
||||
*/
|
||||
|
||||
/**
|
||||
* Calculates an OpenCV match descriptor of an image, which can be used
|
||||
* for brute-force matching.
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
|
||||
* for more details.
|
||||
* The caller is responsible for releasing the returned objects.
|
||||
*
|
||||
* @param {OpenCVBindings['Mat']} img Image data
|
||||
* @param {OpenCVBindings['FeatureDetector']} detector OpenCV feature detector instance
|
||||
*
|
||||
* @returns {MatchComputationResult}
|
||||
*/
|
||||
function detectAndCompute(img, detector) {
|
||||
const keyPoints = new cv.KeyPointVector();
|
||||
const descriptor = new cv.Mat();
|
||||
detector.detect(img, keyPoints);
|
||||
detector.compute(img, keyPoints, descriptor);
|
||||
return {
|
||||
keyPoints,
|
||||
descriptor,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculated the bounding rect coordinates for the array of matching points
|
||||
*
|
||||
* @param {Point[]} matchedPoints Array of matching points
|
||||
* @returns {Rect} The matching bounding rect or a zero rect if no match
|
||||
* can be found.
|
||||
*/
|
||||
function calculateMatchedRect(matchedPoints) {
|
||||
if (matchedPoints.length < 2) {
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const pointsSortedByDistance = matchedPoints
|
||||
.map((point) => [Math.sqrt(point.x * point.x + point.y * point.y), point])
|
||||
// @ts-ignore TS does not understand this, maybe its bug
|
||||
.sort((pair1, pair2) => pair1[0] >= pair2[0])
|
||||
.map((/** @type {[number, Point]} */ pair) => pair[1]);
|
||||
const firstPoint = /** @type {Point} */ (_.head(pointsSortedByDistance));
|
||||
const lastPoint = /** @type {Point} */ (_.last(pointsSortedByDistance));
|
||||
const topLeftPoint = {
|
||||
x: firstPoint.x <= lastPoint.x ? firstPoint.x : lastPoint.x,
|
||||
y: firstPoint.y <= lastPoint.y ? firstPoint.y : lastPoint.y,
|
||||
};
|
||||
const bottomRightPoint = {
|
||||
x: firstPoint.x >= lastPoint.x ? firstPoint.x : lastPoint.x,
|
||||
y: firstPoint.y >= lastPoint.y ? firstPoint.y : lastPoint.y,
|
||||
};
|
||||
return {
|
||||
x: topLeftPoint.x,
|
||||
y: topLeftPoint.y,
|
||||
width: bottomRightPoint.x - topLeftPoint.x,
|
||||
height: bottomRightPoint.y - topLeftPoint.y,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws a rectangle on the given image matrix
|
||||
*
|
||||
* @param {OpenCVBindings['Mat']} mat The source image
|
||||
* @param {Rect} region The region to highlight
|
||||
*
|
||||
* @returns {OpenCVBindings['Mat']} The same image with the rectangle on it
|
||||
*/
|
||||
function highlightRegion(mat, region) {
|
||||
if (region.width <= 0 || region.height <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
// highlight in red
|
||||
const color = pool.add(new cv.Scalar(255, 0, 0, 255));
|
||||
const thickness = 2;
|
||||
const topLeft = pool.add(new cv.Point(region.x, region.y));
|
||||
const botRight = pool.add(new cv.Point(region.x + region.width, region.y + region.height));
|
||||
cv.rectangle(mat, topLeft, botRight, color, thickness, cv.LINE_8, 0);
|
||||
return mat;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef MatchingOptions
|
||||
* @property {string} [detectorName='ORB'] One of possible OpenCV feature detector names
|
||||
* from keys of the `AVAILABLE_DETECTORS` object.
|
||||
* Some of these methods (FAST, AGAST, GFTT, FAST, SIFT and MSER) are not available
|
||||
* in the default OpenCV installation and have to be enabled manually before
|
||||
* library compilation.
|
||||
* @property {string} [matchFunc='BruteForce'] The name of the matching function.
|
||||
* Should be one of the keys of the `AVAILABLE_MATCHING_FUNCTIONS` object.
|
||||
* @property {number|Function?} [goodMatchesFactor] The maximum count of "good" matches
|
||||
* (e. g. with minimal distances) or a function, which accepts 3 arguments: the current distance,
|
||||
* minimal distance, maximum distance and returns true or false to include or exclude the match.
|
||||
* @property {boolean} [visualize=false] Whether to return the resulting visualization
|
||||
* as an image (useful for debugging purposes)
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef MatchingResult
|
||||
* @property {number} count The count of matched edges on both images.
|
||||
* The more matching edges there are no both images the more similar they are.
|
||||
* @property {number} totalCount The total count of matched edges on both images.
|
||||
* It is equal to `count` if `goodMatchesFactor` does not limit the matches,
|
||||
* otherwise it contains the total count of matches before `goodMatchesFactor` is
|
||||
* applied.
|
||||
* @property {Buffer?} [visualization] The visualization of the matching result
|
||||
* represented as PNG image buffer. This visualization looks like
|
||||
* https://user-images.githubusercontent.com/31125521/29702731-c79e3142-8972-11e7-947e-db109d415469.jpg
|
||||
* @property {Point[]} points1 The array of matching points on the first image
|
||||
* @property {Rect} rect1 The bounding rect for the `matchedPoints1` set or a zero rect
|
||||
* if not enough matching points are found
|
||||
* @property {Point[]} points2 The array of matching points on the second image
|
||||
* @property {Rect} rect2 The bounding rect for the `matchedPoints2` set or a zero rect
|
||||
* if not enough matching points are found
|
||||
*/
|
||||
|
||||
/**
|
||||
* Calculates the count of common edges between two images.
|
||||
* The images might be rotated or resized relatively to each other.
|
||||
*
|
||||
* @param {Buffer} img1Data The data of the first image packed into a NodeJS buffer
|
||||
* @param {Buffer} img2Data The data of the second image packed into a NodeJS buffer
|
||||
* @param {MatchingOptions} [options={}] Set of matching options
|
||||
*
|
||||
* @returns {Promise<MatchingResult>} Matching result
|
||||
* @throws {Error} If `detectorName` value is unknown.
|
||||
*/
|
||||
async function getImagesMatches(img1Data, img2Data, options = {}) {
|
||||
await initOpenCv();
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const {
|
||||
detectorName = 'ORB',
|
||||
visualize = false,
|
||||
goodMatchesFactor,
|
||||
matchFunc = 'BruteForce',
|
||||
} = options;
|
||||
if (!_.includes(_.keys(AVAILABLE_DETECTORS), detectorName)) {
|
||||
throw new Error(
|
||||
`'${detectorName}' detector is unknown. ` +
|
||||
`Only ${JSON.stringify(_.keys(AVAILABLE_DETECTORS))} detectors are supported.`
|
||||
);
|
||||
}
|
||||
if (!_.includes(_.keys(AVAILABLE_MATCHING_FUNCTIONS), matchFunc)) {
|
||||
throw new Error(
|
||||
`'${matchFunc}' matching function is unknown. ` +
|
||||
`Only ${JSON.stringify(
|
||||
_.keys(AVAILABLE_MATCHING_FUNCTIONS)
|
||||
)} matching functions are supported.`
|
||||
);
|
||||
}
|
||||
|
||||
const detector = pool.add(new cv[AVAILABLE_DETECTORS[detectorName]]());
|
||||
const [img1, img2] = (await B.all([cvMatFromImage(img1Data), cvMatFromImage(img2Data)]))
|
||||
.map((x) => pool.add(x));
|
||||
const result1 = detectAndCompute(img1, detector);
|
||||
pool.add(result1.keyPoints, result1.descriptor);
|
||||
const result2 = detectAndCompute(img2, detector);
|
||||
pool.add(result2.keyPoints, result2.descriptor);
|
||||
const matcher = pool.add(new cv.DescriptorMatcher(AVAILABLE_MATCHING_FUNCTIONS[matchFunc]));
|
||||
const matchesVec = pool.add(new cv.DMatchVector());
|
||||
matcher.match(result1.descriptor, result2.descriptor, matchesVec);
|
||||
const totalCount = matchesVec.size();
|
||||
if (totalCount < 1) {
|
||||
throw new Error(
|
||||
`Could not find any matches between images. Double-check orientation, ` +
|
||||
`resolution, or use another detector or matching function.`
|
||||
);
|
||||
}
|
||||
let matches = [];
|
||||
for (let i = 0; i < totalCount; i++) {
|
||||
matches.push(matchesVec.get(i));
|
||||
}
|
||||
|
||||
const hasGoodMatchesFactor =
|
||||
_.isFunction(goodMatchesFactor) ||
|
||||
(_.isNumber(goodMatchesFactor) && !_.isNaN(goodMatchesFactor));
|
||||
|
||||
if (hasGoodMatchesFactor) {
|
||||
if (_.isFunction(goodMatchesFactor)) {
|
||||
const distances = matches.map((match) => match.distance);
|
||||
const minDistance = _.min(distances);
|
||||
const maxDistance = _.max(distances);
|
||||
matches = matches.filter((match) =>
|
||||
goodMatchesFactor(match.distance, minDistance, maxDistance)
|
||||
);
|
||||
} else if (matches.length > goodMatchesFactor) {
|
||||
matches = matches
|
||||
.sort((match1, match2) => match1.distance - match2.distance)
|
||||
.slice(0, goodMatchesFactor);
|
||||
}
|
||||
}
|
||||
|
||||
const extractPoint = (keyPoints, indexPropertyName) => (match) => {
|
||||
const {pt, point} = keyPoints.get(match[indexPropertyName]);
|
||||
// https://github.com/justadudewhohacks/opencv4nodejs/issues/584
|
||||
return pt || point;
|
||||
};
|
||||
const points1 = matches.map(extractPoint(result1.keyPoints, 'queryIdx'));
|
||||
const rect1 = calculateMatchedRect(points1);
|
||||
const points2 = matches.map(extractPoint(result2.keyPoints, 'trainIdx'));
|
||||
const rect2 = calculateMatchedRect(points2);
|
||||
|
||||
const result = {
|
||||
points1,
|
||||
rect1,
|
||||
points2,
|
||||
rect2,
|
||||
totalCount,
|
||||
count: matches.length,
|
||||
};
|
||||
if (visualize) {
|
||||
const goodMatchesVec = pool.add(new cv.DMatchVector());
|
||||
for (const match of matches) {
|
||||
goodMatchesVec.push_back(match);
|
||||
}
|
||||
const visualization = pool.add(new cv.Mat());
|
||||
const color = pool.add(new cv.Scalar(0, 255, 0, 255));
|
||||
cv.drawMatches(
|
||||
img1,
|
||||
result1.keyPoints,
|
||||
img2,
|
||||
result2.keyPoints,
|
||||
goodMatchesVec,
|
||||
visualization,
|
||||
color
|
||||
);
|
||||
highlightRegion(visualization, rect1);
|
||||
highlightRegion(visualization, {
|
||||
x: img1.cols + rect2.x,
|
||||
y: rect2.y,
|
||||
width: rect2.width,
|
||||
height: rect2.height,
|
||||
});
|
||||
result.visualization = await cvMatToPng(visualization);
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef SimilarityOptions
|
||||
* @property {boolean} [visualize=false] Whether to return the resulting visalization
|
||||
* as an image (useful for debugging purposes)
|
||||
* @property {string} [method='TM_CCOEFF_NORMED'] The name of the template matching method.
|
||||
* Acceptable values are:
|
||||
* - `TM_CCOEFF`
|
||||
* - `TM_CCOEFF_NORMED` (default)
|
||||
* - `TM_CCORR`
|
||||
* - `TM_CCORR_NORMED`
|
||||
* - `TM_SQDIFF`
|
||||
* - `TM_SQDIFF_NORMED`
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
* for more details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef SimilarityResult
|
||||
* @property {number} score The similarity score as a float number in range [0.0, 1.0].
|
||||
* 1.0 is the highest score (means both images are totally equal).
|
||||
* @property {Buffer?} [visualization] The visualization of the matching result
|
||||
* represented as PNG image buffer. This image includes both input pictures where
|
||||
* difference regions are highlighted with rectangles.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Calculates the similarity score between two images.
|
||||
* It is expected, that both images have the same resolution.
|
||||
*
|
||||
* @param {Buffer} img1Data The data of the first image packed into a NodeJS buffer
|
||||
* @param {Buffer} img2Data The data of the second image packed into a NodeJS buffer
|
||||
* @param {SimilarityOptions} [options={}] Set of similarity calculation options
|
||||
*
|
||||
* @returns {Promise<SimilarityResult>} The calculation result
|
||||
* @throws {Error} If the given images have different resolution.
|
||||
*/
|
||||
async function getImagesSimilarity(img1Data, img2Data, options = {}) {
|
||||
await initOpenCv();
|
||||
|
||||
const {method = DEFAULT_MATCHING_METHOD, visualize = false} = options;
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const [template, reference] = (await B.all([cvMatFromImage(img1Data), cvMatFromImage(img2Data)]))
|
||||
.map((x) => pool.add(x));
|
||||
if (template.rows !== reference.rows || template.cols !== reference.cols) {
|
||||
throw new Error(
|
||||
'Both images are expected to have the same size in order to ' +
|
||||
'calculate the similarity score.'
|
||||
);
|
||||
}
|
||||
template.convertTo(template, cv.CV_8UC3);
|
||||
reference.convertTo(reference, cv.CV_8UC3);
|
||||
|
||||
const matched = pool.add(new cv.Mat());
|
||||
cv.matchTemplate(reference, template, matched, toMatchingMethod(method));
|
||||
const minMax = cv.minMaxLoc(matched);
|
||||
const result = {
|
||||
score: minMax.maxVal,
|
||||
};
|
||||
|
||||
if (visualize) {
|
||||
const resultMat = pool.add(new cv.Mat(template.rows, template.cols * 2, cv.CV_8UC3));
|
||||
const bothImages = pool.add(new cv.MatVector());
|
||||
bothImages.push_back(reference);
|
||||
bothImages.push_back(template);
|
||||
cv.hconcat(bothImages, resultMat);
|
||||
|
||||
const mask = pool.add(new cv.Mat());
|
||||
cv.absdiff(reference, template, mask);
|
||||
cv.cvtColor(mask, mask, cv.COLOR_BGR2GRAY, 0);
|
||||
|
||||
cv.threshold(mask, mask, 128, 255, cv.THRESH_BINARY | cv.THRESH_OTSU);
|
||||
const contours = pool.add(new cv.MatVector());
|
||||
const hierarchy = pool.add(new cv.Mat());
|
||||
cv.findContours(mask, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
|
||||
|
||||
for (let i = 0; i < contours.size(); i++) {
|
||||
const boundingRect = pool.add(cv.boundingRect(contours.get(i)));
|
||||
highlightRegion(resultMat, boundingRect);
|
||||
highlightRegion(resultMat, {
|
||||
x: reference.cols + boundingRect.x,
|
||||
y: boundingRect.y,
|
||||
width: boundingRect.width,
|
||||
height: boundingRect.height,
|
||||
});
|
||||
}
|
||||
result.visualization = await cvMatToPng(resultMat);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef OccurrenceOptions
|
||||
* @property {boolean} [visualize=false] Whether to return the resulting visalization
|
||||
* as an image (useful for debugging purposes)
|
||||
* @property {number} [threshold=0.5] At what normalized threshold to reject
|
||||
* a match
|
||||
* @property {number|boolean} [multiple=false] find multiple matches in the image
|
||||
* @property {number} [matchNeighbourThreshold=10] The pixel distance between matches we consider
|
||||
* to be part of the same template match
|
||||
* @property {TemplateMatchingMethod} [method='TM_CCOEFF_NORMED'] The name of the template matching method.
|
||||
* Acceptable values are:
|
||||
* - `TM_CCOEFF`
|
||||
* - `TM_CCOEFF_NORMED` (default)
|
||||
* - `TM_CCORR`
|
||||
* - `TM_CCORR_NORMED`
|
||||
* - `TM_SQDIFF`
|
||||
* - `TM_SQDIFF_NORMED`
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
* for more details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef OccurrenceResult
|
||||
* @property {import('@appium/types').Rect} rect The region of the partial image occurrence
|
||||
* on the full image
|
||||
* @property {Buffer?} [visualization] The visualization of the matching result
|
||||
* represented as PNG image buffer. On this image the matching
|
||||
* region is highlighted with a rectangle. If the multiple option is passed,
|
||||
* all results are highlighted here.
|
||||
* @property {number} score The similarity score as a float number in range [0.0, 1.0].
|
||||
* 1.0 is the highest score (means both images are totally equal).
|
||||
* @property {OccurrenceResult[]} [multiple] The array of matching OccurrenceResults
|
||||
* - only when multiple option is passed
|
||||
*/
|
||||
|
||||
/**
|
||||
* Calculates the occurrence position of a partial image in the full
|
||||
* image.
|
||||
*
|
||||
* @param {Buffer} fullImgData The data of the full image packed into a NodeJS buffer
|
||||
* @param {Buffer} partialImgData The data of the partial image packed into a NodeJS buffer
|
||||
* @param {OccurrenceOptions} [options] Set of occurrence calculation options
|
||||
*
|
||||
* @returns {Promise<OccurrenceResult>}
|
||||
* @throws {Error} If no occurrences of the partial image can be found in the full image
|
||||
*/
|
||||
async function getImageOccurrence(fullImgData, partialImgData, options = {}) {
|
||||
await initOpenCv();
|
||||
|
||||
const {
|
||||
visualize = false,
|
||||
threshold = DEFAULT_MATCH_THRESHOLD,
|
||||
multiple = false,
|
||||
matchNeighbourThreshold = MATCH_NEIGHBOUR_THRESHOLD,
|
||||
method = DEFAULT_MATCHING_METHOD,
|
||||
} = options;
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const [fullImg, partialImg] = await B.all([cvMatFromImage(fullImgData), cvMatFromImage(partialImgData)])
|
||||
.map((x) => pool.add(x));
|
||||
const matched = pool.add(new cv.Mat());
|
||||
const results = [];
|
||||
let visualization = null;
|
||||
|
||||
try {
|
||||
cv.matchTemplate(fullImg, partialImg, matched, toMatchingMethod(method));
|
||||
const minMax = cv.minMaxLoc(matched);
|
||||
|
||||
if (multiple) {
|
||||
const matches = [];
|
||||
for (let row = 0; row < matched.rows; row++) {
|
||||
for (let col = 0; col < matched.cols; col++) {
|
||||
const score = matched.floatAt(row, col);
|
||||
if (score >= threshold) {
|
||||
matches.push({score, x: col, y: row});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nearMatches = filterNearMatches(matches, matchNeighbourThreshold);
|
||||
|
||||
for (const {x, y, score} of nearMatches) {
|
||||
results.push({
|
||||
score,
|
||||
rect: {
|
||||
x,
|
||||
y,
|
||||
width: partialImg.cols,
|
||||
height: partialImg.rows,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else if (minMax.maxVal >= threshold) {
|
||||
const {x, y} = method.includes('SQDIFF') ? minMax.minLoc : minMax.maxLoc;
|
||||
results.push({
|
||||
score: minMax.maxVal,
|
||||
rect: {
|
||||
x,
|
||||
y,
|
||||
width: partialImg.cols,
|
||||
height: partialImg.rows,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (_.isEmpty(results)) {
|
||||
// Below error message, `Cannot find any occurrences` is referenced in find by image
|
||||
throw new Error(
|
||||
`Match threshold: ${threshold}. Highest match value found was ${minMax.maxVal}`
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
// Below error message, `Cannot find any occurrences` is referenced in find by image
|
||||
throw new Error(
|
||||
`Cannot find any occurrences of the partial image in the full image. ` +
|
||||
`Original error: ${e.message}`
|
||||
);
|
||||
}
|
||||
|
||||
if (visualize) {
|
||||
const fullHighlightedImage = pool.add(fullImg.clone());
|
||||
|
||||
const visualizePromises = [];
|
||||
for (const result of results) {
|
||||
const singleHighlightedImage = pool.add(fullImg.clone());
|
||||
|
||||
highlightRegion(singleHighlightedImage, result.rect);
|
||||
highlightRegion(fullHighlightedImage, result.rect);
|
||||
visualizePromises.push(cvMatToPng(singleHighlightedImage));
|
||||
}
|
||||
let restPngBuffers = [];
|
||||
[visualization, ...restPngBuffers] = await B.all(
|
||||
[cvMatToPng(fullHighlightedImage), ...visualizePromises]
|
||||
);
|
||||
for (const [result, pngBuffer] of _.zip(results, restPngBuffers)) {
|
||||
// @ts-ignore This is fine
|
||||
result.visualization = pngBuffer;
|
||||
}
|
||||
}
|
||||
return {
|
||||
rect: results[0].rect,
|
||||
score: results[0].score,
|
||||
visualization,
|
||||
multiple: results,
|
||||
};
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an opencv image matrix into a PNG buffer
|
||||
*
|
||||
* @param {OpenCVBindings['Mat']} mat OpenCV image matrix
|
||||
* @return {Promise<Buffer>} PNG image data buffer
|
||||
*/
|
||||
async function cvMatToPng(mat) {
|
||||
return await sharp(Buffer.from(mat.data), {
|
||||
raw: {
|
||||
width: mat.cols,
|
||||
height: mat.rows,
|
||||
channels: 4,
|
||||
}
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Take an image buffer and return a cv.Mat
|
||||
*
|
||||
* @param {Buffer} img image data buffer. All image formats available for
|
||||
* https://www.npmjs.com/package/sharp node library are supported.
|
||||
* @return {Promise<OpenCVBindings['Mat']>} OpenCV image matrix
|
||||
*/
|
||||
async function cvMatFromImage(img) {
|
||||
const {data, info} = await sharp(img)
|
||||
.ensureAlpha()
|
||||
.raw()
|
||||
.toBuffer({resolveWithObject: true});
|
||||
const {width, height} = info;
|
||||
return cv.matFromImageData({data, width, height});
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} Match
|
||||
* @property {number} score
|
||||
* @property {number} x
|
||||
* @property {number} y
|
||||
*/
|
||||
|
||||
/**
|
||||
* Filter out match results which have a matched neighbour
|
||||
*
|
||||
* @param {Match[]} nonZeroMatchResults matrix of image match results
|
||||
* @param {number} matchNeighbourThreshold The pixel distance within which we
|
||||
* consider an element being a neighbour of an existing match
|
||||
* @returns {Match[]} the filtered array of matched points
|
||||
*/
|
||||
function filterNearMatches(nonZeroMatchResults, matchNeighbourThreshold) {
|
||||
return nonZeroMatchResults.reduce((acc, element) => {
|
||||
if (!acc.some((match) => distance(match, element) <= matchNeighbourThreshold)) {
|
||||
// @ts-ignore TS cannot properly understand types here
|
||||
acc.push(element);
|
||||
}
|
||||
return acc;
|
||||
}, []);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the distance between two points
|
||||
*
|
||||
* @param {Point} point1 The first point
|
||||
* @param {Point} point2 The second point
|
||||
* @return {number} the distance
|
||||
*/
|
||||
function distance(point1, point2) {
|
||||
const a2 = Math.pow(point1.x - point2.x, 2);
|
||||
const b2 = Math.pow(point1.y - point2.y, 2);
|
||||
return Math.sqrt(a2 + b2);
|
||||
}
|
||||
|
||||
export {getImagesMatches, getImagesSimilarity, getImageOccurrence, initOpenCv};
|
||||
|
||||
/**
|
||||
* @typedef OpenCVBindings
|
||||
* @property {any} Mat
|
||||
* @property {any} KeyPointVector
|
||||
* @property {any} FeatureDetector
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {'TM_CCOEFF'|'TM_CCOEFF_NORMED'|'TM_CCORR'|'TM_CCORR_NORMED'|'TM_SQDIFF'|'TMSQDIFF_NORMED'} TemplateMatchingMethod
|
||||
*/
|
||||
@@ -0,0 +1,609 @@
|
||||
import _ from 'lodash';
|
||||
import {Buffer} from 'node:buffer';
|
||||
import B from 'bluebird';
|
||||
import sharp from 'sharp';
|
||||
import {OpenCvAutoreleasePool} from './autorelease-pool';
|
||||
import type {
|
||||
Point,
|
||||
Rect,
|
||||
Match,
|
||||
MatchComputationResult,
|
||||
MatchingOptions,
|
||||
MatchingResult,
|
||||
SimilarityOptions,
|
||||
SimilarityResult,
|
||||
OccurrenceOptions,
|
||||
OccurrenceResult,
|
||||
TemplateMatchingMethod,
|
||||
OpenCVBindings,
|
||||
} from './types';
|
||||
|
||||
let cv: OpenCVBindings | undefined;
|
||||
|
||||
export const DEFAULT_MATCH_THRESHOLD = 0.5;
|
||||
export const MATCH_NEIGHBOUR_THRESHOLD = 10;
|
||||
|
||||
export const AVAILABLE_DETECTORS = {
|
||||
AKAZE: 'AKAZE',
|
||||
AGAST: 'AgastFeatureDetector',
|
||||
BRISK: 'BRISK',
|
||||
FAST: 'FastFeatureDetector',
|
||||
GFTT: 'GFTTDetector',
|
||||
KAZE: 'KAZE',
|
||||
MSER: 'MSER',
|
||||
ORB: 'ORB',
|
||||
} as const;
|
||||
|
||||
export const AVAILABLE_MATCHING_FUNCTIONS = {
|
||||
FlannBased: 'FlannBased',
|
||||
BruteForce: 'BruteForce',
|
||||
BruteForceL1: 'BruteForce-L1',
|
||||
BruteForceHamming: 'BruteForce-Hamming',
|
||||
BruteForceHammingLut: 'BruteForce-HammingLUT',
|
||||
BruteForceSL2: 'BruteForce-SL2',
|
||||
} as const;
|
||||
|
||||
const MATCHING_METHODS = [
|
||||
'TM_CCOEFF',
|
||||
'TM_CCOEFF_NORMED',
|
||||
'TM_CCORR',
|
||||
'TM_CCORR_NORMED',
|
||||
'TM_SQDIFF',
|
||||
'TM_SQDIFF_NORMED',
|
||||
] as const;
|
||||
|
||||
const DEFAULT_MATCHING_METHOD: TemplateMatchingMethod = 'TM_CCOEFF_NORMED';
|
||||
|
||||
// Public exported functions
|
||||
|
||||
/**
|
||||
* Initializes the OpenCV bindings library.
|
||||
* Spins until the opencv-bindings module is fully loaded.
|
||||
* You only need to explicitly call this if you want to use your own opencv
|
||||
* methods that are not included in this module.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import {initOpenCv} from '@appium/opencv';
|
||||
* await initOpenCv();
|
||||
* ```
|
||||
*/
|
||||
export async function initOpenCv(): Promise<void> {
|
||||
cv = require('opencv-bindings') as OpenCVBindings;
|
||||
while (!cv.getBuildInformation) {
|
||||
await B.delay(500);
|
||||
}
|
||||
// opencv-bindings sets a global unhandledRejection handler of an abort, which we don't want, so
|
||||
// undo it here. https://github.com/opencv/opencv/issues/21481
|
||||
process.removeAllListeners('unhandledRejection');
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the count of common edges between two images.
|
||||
* The images might be rotated or resized relatively to each other.
|
||||
* This method uses feature-based matching which is useful when images are rotated/scaled.
|
||||
*
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_feature2d/py_matcher/py_matcher.html
|
||||
* for more details on feature-based matching.
|
||||
*
|
||||
* @param img1Data - The data of the first image packed into a NodeJS buffer
|
||||
* @param img2Data - The data of the second image packed into a NodeJS buffer
|
||||
* @param options - Set of matching options (see {@link MatchingOptions})
|
||||
* @returns Matching result containing count, totalCount, points, rects, and optionally visualization
|
||||
* @throws {Error} If `detectorName` or `matchFunc` value is unknown, or if no matches can be found between images
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import {getImagesMatches} from '@appium/opencv';
|
||||
* import {fs} from '@appium/support';
|
||||
*
|
||||
* const image1 = await fs.readFile('image1.jpg');
|
||||
* const image2 = await fs.readFile('image2.jpg');
|
||||
* const {points1, rect1, points2, rect2, totalCount, count} = await getImagesMatches(image1, image2);
|
||||
* ```
|
||||
*/
|
||||
export async function getImagesMatches(
|
||||
img1Data: Buffer,
|
||||
img2Data: Buffer,
|
||||
options: MatchingOptions = {}
|
||||
): Promise<MatchingResult> {
|
||||
await initOpenCv();
|
||||
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized.');
|
||||
}
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const {
|
||||
detectorName = 'ORB',
|
||||
visualize = false,
|
||||
goodMatchesFactor,
|
||||
matchFunc = 'BruteForce',
|
||||
} = options;
|
||||
if (!_.includes(_.keys(AVAILABLE_DETECTORS), detectorName)) {
|
||||
throw new Error(
|
||||
`'${detectorName}' detector is unknown. ` +
|
||||
`Only ${JSON.stringify(_.keys(AVAILABLE_DETECTORS))} detectors are supported.`
|
||||
);
|
||||
}
|
||||
if (!_.includes(_.keys(AVAILABLE_MATCHING_FUNCTIONS), matchFunc)) {
|
||||
throw new Error(
|
||||
`'${matchFunc}' matching function is unknown. ` +
|
||||
`Only ${JSON.stringify(_.keys(AVAILABLE_MATCHING_FUNCTIONS))} matching functions are supported.`
|
||||
);
|
||||
}
|
||||
|
||||
const detector = pool.add(new cv[AVAILABLE_DETECTORS[detectorName]]());
|
||||
const img1Promise = cvMatFromImage(img1Data);
|
||||
const img2Promise = cvMatFromImage(img2Data);
|
||||
const [img1Raw, img2Raw] = await B.all([img1Promise, img2Promise]);
|
||||
const img1 = pool.add(img1Raw);
|
||||
const img2 = pool.add(img2Raw);
|
||||
const result1 = detectAndCompute(img1, detector);
|
||||
pool.add(result1.keyPoints);
|
||||
pool.add(result1.descriptor);
|
||||
const result2 = detectAndCompute(img2, detector);
|
||||
pool.add(result2.keyPoints);
|
||||
pool.add(result2.descriptor);
|
||||
const matcher = pool.add(new cv.DescriptorMatcher(AVAILABLE_MATCHING_FUNCTIONS[matchFunc]));
|
||||
const matchesVec = pool.add(new cv.DMatchVector());
|
||||
matcher.match(result1.descriptor, result2.descriptor, matchesVec);
|
||||
const totalCount = matchesVec.size();
|
||||
if (totalCount < 1) {
|
||||
throw new Error(
|
||||
`Could not find any matches between images. Double-check orientation, ` +
|
||||
`resolution, or use another detector or matching function.`
|
||||
);
|
||||
}
|
||||
let matches: any[] = [];
|
||||
for (let i = 0; i < totalCount; i++) {
|
||||
matches.push(matchesVec.get(i));
|
||||
}
|
||||
|
||||
const hasGoodMatchesFactor =
|
||||
_.isFunction(goodMatchesFactor) ||
|
||||
(_.isNumber(goodMatchesFactor) && !_.isNaN(goodMatchesFactor));
|
||||
|
||||
if (hasGoodMatchesFactor) {
|
||||
if (_.isFunction(goodMatchesFactor)) {
|
||||
const distances = matches.map((match: any) => match.distance);
|
||||
const minDistance = _.min(distances);
|
||||
const maxDistance = _.max(distances);
|
||||
if (minDistance !== undefined && maxDistance !== undefined) {
|
||||
matches = matches.filter((match: any) =>
|
||||
goodMatchesFactor(match.distance, minDistance, maxDistance)
|
||||
);
|
||||
}
|
||||
} else if (_.isNumber(goodMatchesFactor) && matches.length > goodMatchesFactor) {
|
||||
matches = matches
|
||||
.sort((match1: any, match2: any) => match1.distance - match2.distance)
|
||||
.slice(0, goodMatchesFactor);
|
||||
}
|
||||
}
|
||||
|
||||
const extractPoint = (keyPoints: any, indexPropertyName: string) => (match: any) => {
|
||||
const {pt, point} = keyPoints.get(match[indexPropertyName]);
|
||||
// https://github.com/justadudewhohacks/opencv4nodejs/issues/584
|
||||
return pt || point;
|
||||
};
|
||||
const points1 = matches.map(extractPoint(result1.keyPoints, 'queryIdx'));
|
||||
const rect1 = calculateMatchedRect(points1);
|
||||
const points2 = matches.map(extractPoint(result2.keyPoints, 'trainIdx'));
|
||||
const rect2 = calculateMatchedRect(points2);
|
||||
|
||||
const result: MatchingResult = {
|
||||
points1,
|
||||
rect1,
|
||||
points2,
|
||||
rect2,
|
||||
totalCount,
|
||||
count: matches.length,
|
||||
};
|
||||
if (visualize) {
|
||||
const goodMatchesVec = pool.add(new cv.DMatchVector());
|
||||
for (const match of matches) {
|
||||
goodMatchesVec.push_back(match);
|
||||
}
|
||||
const visualization = pool.add(new cv.Mat());
|
||||
const color = pool.add(new cv.Scalar(0, 255, 0, 255));
|
||||
cv.drawMatches(
|
||||
img1,
|
||||
result1.keyPoints,
|
||||
img2,
|
||||
result2.keyPoints,
|
||||
goodMatchesVec,
|
||||
visualization,
|
||||
color
|
||||
);
|
||||
highlightRegion(visualization, rect1);
|
||||
highlightRegion(visualization, {
|
||||
x: img1.cols + rect2.x,
|
||||
y: rect2.y,
|
||||
width: rect2.width,
|
||||
height: rect2.height,
|
||||
});
|
||||
result.visualization = await cvMatToPng(visualization);
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the similarity score between two images.
|
||||
* It is expected that both images have the same resolution.
|
||||
* This method uses template matching to compare images pixel-by-pixel.
|
||||
*
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
* for more details on template matching.
|
||||
*
|
||||
* @param img1Data - The data of the first image packed into a NodeJS buffer
|
||||
* @param img2Data - The data of the second image packed into a NodeJS buffer
|
||||
* @param options - Set of similarity calculation options (see {@link SimilarityOptions})
|
||||
* @returns Similarity result containing score (float in range [0.0, 1.0], where 1.0 means images are totally equal) and optionally visualization
|
||||
* @throws {Error} If the given images have different resolution
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import {getImagesSimilarity} from '@appium/opencv';
|
||||
* import {fs} from '@appium/support';
|
||||
*
|
||||
* const image1 = await fs.readFile('image1.jpg');
|
||||
* const image2 = await fs.readFile('image2.jpg');
|
||||
* const {score} = await getImagesSimilarity(image1, image2);
|
||||
* ```
|
||||
*/
|
||||
export async function getImagesSimilarity(
|
||||
img1Data: Buffer,
|
||||
img2Data: Buffer,
|
||||
options: SimilarityOptions = {}
|
||||
): Promise<SimilarityResult> {
|
||||
await initOpenCv();
|
||||
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized.');
|
||||
}
|
||||
|
||||
const {method = DEFAULT_MATCHING_METHOD, visualize = false} = options;
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const templatePromise = cvMatFromImage(img1Data);
|
||||
const referencePromise = cvMatFromImage(img2Data);
|
||||
const [templateRaw, referenceRaw] = await B.all([templatePromise, referencePromise]);
|
||||
const template = pool.add(templateRaw);
|
||||
const reference = pool.add(referenceRaw);
|
||||
if (template.rows !== reference.rows || template.cols !== reference.cols) {
|
||||
throw new Error(
|
||||
'Both images are expected to have the same size in order to ' +
|
||||
'calculate the similarity score.'
|
||||
);
|
||||
}
|
||||
template.convertTo(template, cv.CV_8UC3);
|
||||
reference.convertTo(reference, cv.CV_8UC3);
|
||||
|
||||
const matched = pool.add(new cv.Mat());
|
||||
cv.matchTemplate(reference, template, matched, toMatchingMethod(method));
|
||||
const minMax = cv.minMaxLoc(matched);
|
||||
const result: SimilarityResult = {
|
||||
score: minMax.maxVal,
|
||||
};
|
||||
|
||||
if (visualize) {
|
||||
const resultMat = pool.add(new cv.Mat(template.rows, template.cols * 2, cv.CV_8UC3));
|
||||
const bothImages = pool.add(new cv.MatVector());
|
||||
bothImages.push_back(reference);
|
||||
bothImages.push_back(template);
|
||||
cv.hconcat(bothImages, resultMat);
|
||||
|
||||
const mask = pool.add(new cv.Mat());
|
||||
cv.absdiff(reference, template, mask);
|
||||
cv.cvtColor(mask, mask, cv.COLOR_BGR2GRAY, 0);
|
||||
|
||||
cv.threshold(mask, mask, 128, 255, cv.THRESH_BINARY | cv.THRESH_OTSU);
|
||||
const contours = pool.add(new cv.MatVector());
|
||||
const hierarchy = pool.add(new cv.Mat());
|
||||
cv.findContours(mask, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
|
||||
|
||||
for (let i = 0; i < contours.size(); i++) {
|
||||
const boundingRect = pool.add(cv.boundingRect(contours.get(i)));
|
||||
highlightRegion(resultMat, boundingRect);
|
||||
highlightRegion(resultMat, {
|
||||
x: reference.cols + boundingRect.x,
|
||||
y: boundingRect.y,
|
||||
width: boundingRect.width,
|
||||
height: boundingRect.height,
|
||||
});
|
||||
}
|
||||
result.visualization = await cvMatToPng(resultMat);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the occurrence position of a partial image in the full image.
|
||||
* This method uses template matching to find where the partial image appears in the full image.
|
||||
*
|
||||
* Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
|
||||
* for more details on template matching.
|
||||
*
|
||||
* @param fullImgData - The data of the full image packed into a NodeJS buffer
|
||||
* @param partialImgData - The data of the partial image packed into a NodeJS buffer
|
||||
* @param options - Set of occurrence calculation options (see {@link OccurrenceOptions})
|
||||
* @returns Occurrence result containing rect (region of the partial image occurrence on the full image), score (similarity score as float in range [0.0, 1.0]), optionally visualization, and optionally multiple (array of matching results when multiple option is enabled)
|
||||
* @throws {Error} If no occurrences of the partial image can be found in the full image
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import {getImageOccurrence} from '@appium/opencv';
|
||||
* import {fs} from '@appium/support';
|
||||
*
|
||||
* const fullImage = await fs.readFile('full-image.jpg');
|
||||
* const partialImage = await fs.readFile('partial-image.jpg');
|
||||
* const {rect, score} = await getImageOccurrence(fullImage, partialImage);
|
||||
* ```
|
||||
*/
|
||||
export async function getImageOccurrence(
|
||||
fullImgData: Buffer,
|
||||
partialImgData: Buffer,
|
||||
options: OccurrenceOptions = {}
|
||||
): Promise<OccurrenceResult> {
|
||||
await initOpenCv();
|
||||
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized.');
|
||||
}
|
||||
|
||||
const {
|
||||
visualize = false,
|
||||
threshold = DEFAULT_MATCH_THRESHOLD,
|
||||
multiple = false,
|
||||
matchNeighbourThreshold = MATCH_NEIGHBOUR_THRESHOLD,
|
||||
method = DEFAULT_MATCHING_METHOD,
|
||||
} = options;
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
const fullImgPromise = cvMatFromImage(fullImgData);
|
||||
const partialImgPromise = cvMatFromImage(partialImgData);
|
||||
const [fullImgRaw, partialImgRaw] = await B.all([fullImgPromise, partialImgPromise]);
|
||||
const fullImg = pool.add(fullImgRaw);
|
||||
const partialImg = pool.add(partialImgRaw);
|
||||
const matched = pool.add(new cv.Mat());
|
||||
const results: Array<{score: number; rect: Rect; visualization?: Buffer}> = [];
|
||||
let visualization: Buffer | null = null;
|
||||
|
||||
try {
|
||||
cv.matchTemplate(fullImg, partialImg, matched, toMatchingMethod(method));
|
||||
const minMax = cv.minMaxLoc(matched);
|
||||
|
||||
if (multiple) {
|
||||
const matches: Match[] = [];
|
||||
for (let row = 0; row < matched.rows; row++) {
|
||||
for (let col = 0; col < matched.cols; col++) {
|
||||
const score = matched.floatAt(row, col);
|
||||
if (score >= threshold) {
|
||||
matches.push({score, x: col, y: row});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nearMatches = filterNearMatches(matches, matchNeighbourThreshold);
|
||||
|
||||
for (const {x, y, score} of nearMatches) {
|
||||
results.push({
|
||||
score,
|
||||
rect: {
|
||||
x,
|
||||
y,
|
||||
width: partialImg.cols,
|
||||
height: partialImg.rows,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else if (minMax.maxVal >= threshold) {
|
||||
const {x, y} = method.includes('SQDIFF') ? minMax.minLoc : minMax.maxLoc;
|
||||
results.push({
|
||||
score: minMax.maxVal,
|
||||
rect: {
|
||||
x,
|
||||
y,
|
||||
width: partialImg.cols,
|
||||
height: partialImg.rows,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (_.isEmpty(results)) {
|
||||
// Below error message, `Cannot find any occurrences` is referenced in find by image
|
||||
throw new Error(
|
||||
`Match threshold: ${threshold}. Highest match value found was ${minMax.maxVal}`
|
||||
);
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Below error message, `Cannot find any occurrences` is referenced in find by image
|
||||
throw new Error(
|
||||
`Cannot find any occurrences of the partial image in the full image. ` +
|
||||
`Original error: ${e?.message || String(e)}`
|
||||
);
|
||||
}
|
||||
|
||||
if (visualize) {
|
||||
const fullHighlightedImage = pool.add(fullImg.clone());
|
||||
|
||||
const visualizePromises: Promise<Buffer>[] = [];
|
||||
for (const result of results) {
|
||||
const singleHighlightedImage = pool.add(fullImg.clone());
|
||||
|
||||
highlightRegion(singleHighlightedImage, result.rect);
|
||||
highlightRegion(fullHighlightedImage, result.rect);
|
||||
visualizePromises.push(cvMatToPng(singleHighlightedImage));
|
||||
}
|
||||
let restPngBuffers: Buffer[] = [];
|
||||
[visualization, ...restPngBuffers] = await B.all([
|
||||
cvMatToPng(fullHighlightedImage),
|
||||
...visualizePromises,
|
||||
]);
|
||||
for (const [result, pngBuffer] of _.zip(results, restPngBuffers)) {
|
||||
if (result && pngBuffer) {
|
||||
result.visualization = pngBuffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
rect: results[0].rect,
|
||||
score: results[0].score,
|
||||
visualization,
|
||||
multiple: results,
|
||||
};
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
// Private helper functions
|
||||
|
||||
function toMatchingMethod(name: string): number {
|
||||
if (!MATCHING_METHODS.includes(name as TemplateMatchingMethod)) {
|
||||
throw new Error(
|
||||
`The matching method '${name}' is unknown. ` +
|
||||
`Only the following matching methods are supported: ${MATCHING_METHODS}`
|
||||
);
|
||||
}
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized. Call initOpenCv() first.');
|
||||
}
|
||||
return cv[name];
|
||||
}
|
||||
|
||||
function detectAndCompute(img: any, detector: any): MatchComputationResult {
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized. Call initOpenCv() first.');
|
||||
}
|
||||
const keyPoints = new cv.KeyPointVector();
|
||||
const descriptor = new cv.Mat();
|
||||
detector.detect(img, keyPoints);
|
||||
detector.compute(img, keyPoints, descriptor);
|
||||
return {
|
||||
keyPoints,
|
||||
descriptor,
|
||||
};
|
||||
}
|
||||
|
||||
function calculateMatchedRect(matchedPoints: Point[]): Rect {
|
||||
if (matchedPoints.length < 2) {
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const pointsSortedByDistance = matchedPoints
|
||||
.map((point) => [Math.sqrt(point.x * point.x + point.y * point.y), point] as [number, Point])
|
||||
.sort((pair1, pair2) => pair1[0] - pair2[0])
|
||||
.map((pair) => pair[1]);
|
||||
const firstPoint = _.head(pointsSortedByDistance);
|
||||
const lastPoint = _.last(pointsSortedByDistance);
|
||||
if (!firstPoint || !lastPoint) {
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
};
|
||||
}
|
||||
const topLeftPoint = {
|
||||
x: firstPoint.x <= lastPoint.x ? firstPoint.x : lastPoint.x,
|
||||
y: firstPoint.y <= lastPoint.y ? firstPoint.y : lastPoint.y,
|
||||
};
|
||||
const bottomRightPoint = {
|
||||
x: firstPoint.x >= lastPoint.x ? firstPoint.x : lastPoint.x,
|
||||
y: firstPoint.y >= lastPoint.y ? firstPoint.y : lastPoint.y,
|
||||
};
|
||||
return {
|
||||
x: topLeftPoint.x,
|
||||
y: topLeftPoint.y,
|
||||
width: bottomRightPoint.x - topLeftPoint.x,
|
||||
height: bottomRightPoint.y - topLeftPoint.y,
|
||||
};
|
||||
}
|
||||
|
||||
function highlightRegion(mat: any, region: Rect): any {
|
||||
if (region.width <= 0 || region.height <= 0) {
|
||||
return mat;
|
||||
}
|
||||
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized. Call initOpenCv() first.');
|
||||
}
|
||||
|
||||
const pool = new OpenCvAutoreleasePool();
|
||||
try {
|
||||
// highlight in red
|
||||
const color = pool.add(new cv.Scalar(255, 0, 0, 255));
|
||||
const thickness = 2;
|
||||
const topLeft = pool.add(new cv.Point(region.x, region.y));
|
||||
const botRight = pool.add(new cv.Point(region.x + region.width, region.y + region.height));
|
||||
cv.rectangle(mat, topLeft, botRight, color, thickness, cv.LINE_8, 0);
|
||||
return mat;
|
||||
} finally {
|
||||
pool.drain();
|
||||
}
|
||||
}
|
||||
|
||||
async function cvMatToPng(mat: any): Promise<Buffer> {
|
||||
return await sharp(Buffer.from(mat.data), {
|
||||
raw: {
|
||||
width: mat.cols,
|
||||
height: mat.rows,
|
||||
channels: 4,
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function cvMatFromImage(img: Buffer): Promise<any> {
|
||||
if (!cv) {
|
||||
throw new Error('OpenCV is not initialized. Call initOpenCv() first.');
|
||||
}
|
||||
const {data, info} = await sharp(img)
|
||||
.ensureAlpha()
|
||||
.raw()
|
||||
.toBuffer({resolveWithObject: true});
|
||||
const {width, height} = info;
|
||||
return cv.matFromImageData({data, width, height});
|
||||
}
|
||||
|
||||
function filterNearMatches(nonZeroMatchResults: Match[], matchNeighbourThreshold: number): Match[] {
|
||||
return nonZeroMatchResults.reduce((acc: Match[], element: Match) => {
|
||||
if (!acc.some((match) => distance(match, element) <= matchNeighbourThreshold)) {
|
||||
acc.push(element);
|
||||
}
|
||||
return acc;
|
||||
}, []);
|
||||
}
|
||||
|
||||
function distance(point1: Point, point2: Point): number {
|
||||
const a2 = Math.pow(point1.x - point2.x, 2);
|
||||
const b2 = Math.pow(point1.y - point2.y, 2);
|
||||
return Math.sqrt(a2 + b2);
|
||||
}
|
||||
|
||||
// Re-export types for consumers
|
||||
export type {
|
||||
MatchingOptions,
|
||||
MatchingResult,
|
||||
SimilarityOptions,
|
||||
SimilarityResult,
|
||||
OccurrenceOptions,
|
||||
OccurrenceResult,
|
||||
};
|
||||
@@ -0,0 +1,148 @@
|
||||
import type {Rect as AppiumRect} from '@appium/types';
|
||||
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface Region {
|
||||
left: number;
|
||||
top: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface Rect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type TemplateMatchingMethod =
|
||||
| 'TM_CCOEFF'
|
||||
| 'TM_CCOEFF_NORMED'
|
||||
| 'TM_CCORR'
|
||||
| 'TM_CCORR_NORMED'
|
||||
| 'TM_SQDIFF'
|
||||
| 'TM_SQDIFF_NORMED';
|
||||
|
||||
export interface Match {
|
||||
score: number;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface MatchComputationResult {
|
||||
descriptor: any;
|
||||
keyPoints: any;
|
||||
}
|
||||
|
||||
export type DetectorName = 'AKAZE' | 'AGAST' | 'BRISK' | 'FAST' | 'GFTT' | 'KAZE' | 'MSER' | 'ORB';
|
||||
export type MatchFuncName =
|
||||
| 'FlannBased'
|
||||
| 'BruteForce'
|
||||
| 'BruteForceL1'
|
||||
| 'BruteForceHamming'
|
||||
| 'BruteForceHammingLut'
|
||||
| 'BruteForceSL2';
|
||||
|
||||
export interface MatchingOptions {
|
||||
/**
|
||||
* One of possible OpenCV feature detector names from keys of the `AVAILABLE_DETECTORS` object.
|
||||
* Defaults to `'ORB'`. Some methods (FAST, AGAST, GFTT, SIFT, MSER) are not available in the
|
||||
* default OpenCV installation and have to be enabled manually before library compilation.
|
||||
*/
|
||||
detectorName?: DetectorName;
|
||||
/**
|
||||
* The name of the matching function. Should be one of the keys of the `AVAILABLE_MATCHING_FUNCTIONS` object.
|
||||
* Defaults to `'BruteForce'`.
|
||||
*/
|
||||
matchFunc?: MatchFuncName;
|
||||
/**
|
||||
* The maximum count of "good" matches (e.g. with minimal distances) or a function, which accepts
|
||||
* 3 arguments: the current distance, minimal distance, maximum distance and returns true or false
|
||||
* to include or exclude the match.
|
||||
*/
|
||||
goodMatchesFactor?: number | ((distance: number, minDistance: number, maxDistance: number) => boolean);
|
||||
/**
|
||||
* Whether to return the resulting visualization as an image buffer (useful for debugging purposes).
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
visualize?: boolean;
|
||||
}
|
||||
|
||||
export interface MatchingResult {
|
||||
count: number;
|
||||
totalCount: number;
|
||||
visualization?: Buffer | null;
|
||||
points1: Point[];
|
||||
rect1: Rect;
|
||||
points2: Point[];
|
||||
rect2: Rect;
|
||||
}
|
||||
|
||||
export interface SimilarityOptions {
|
||||
/**
|
||||
* Whether to return the resulting visualization as an image buffer where difference regions are
|
||||
* highlighted with rectangles. Defaults to `false`.
|
||||
*/
|
||||
visualize?: boolean;
|
||||
/**
|
||||
* The name of the template matching method. Acceptable values are: `TM_CCOEFF`, `TM_CCOEFF_NORMED` (default),
|
||||
* `TM_CCORR`, `TM_CCORR_NORMED`, `TM_SQDIFF`, `TM_SQDIFF_NORMED`. Defaults to `'TM_CCOEFF_NORMED'`.
|
||||
*/
|
||||
method?: TemplateMatchingMethod;
|
||||
}
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number;
|
||||
visualization?: Buffer | null;
|
||||
}
|
||||
|
||||
export interface OccurrenceOptions {
|
||||
/**
|
||||
* Whether to return the resulting visualization as an image buffer where the matching region is
|
||||
* highlighted with a rectangle. If the multiple option is passed, all results are highlighted.
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
visualize?: boolean;
|
||||
/**
|
||||
* At what normalized threshold to reject a match. Defaults to `0.5`.
|
||||
*/
|
||||
threshold?: number;
|
||||
/**
|
||||
* Whether to find multiple matches in the image. If `true` or a number, returns all matches above
|
||||
* the threshold. Defaults to `false`.
|
||||
*/
|
||||
multiple?: boolean | number;
|
||||
/**
|
||||
* The pixel distance between matches we consider to be part of the same template match.
|
||||
* Defaults to `10`.
|
||||
*/
|
||||
matchNeighbourThreshold?: number;
|
||||
/**
|
||||
* The name of the template matching method. Acceptable values are: `TM_CCOEFF`, `TM_CCOEFF_NORMED` (default),
|
||||
* `TM_CCORR`, `TM_CCORR_NORMED`, `TM_SQDIFF`, `TM_SQDIFF_NORMED`. Defaults to `'TM_CCOEFF_NORMED'`.
|
||||
*/
|
||||
method?: TemplateMatchingMethod;
|
||||
}
|
||||
|
||||
export interface OccurrenceResult {
|
||||
rect: AppiumRect;
|
||||
visualization?: Buffer | null;
|
||||
score: number;
|
||||
multiple?: OccurrenceResult[];
|
||||
}
|
||||
|
||||
export interface OpenCVBindings {
|
||||
Mat: any;
|
||||
KeyPointVector: any;
|
||||
FeatureDetector: any;
|
||||
Scalar: new (...args: number[]) => any;
|
||||
Point: new (x: number, y: number) => any;
|
||||
DescriptorMatcher: new (matchFunc: string) => any;
|
||||
DMatchVector: new () => any;
|
||||
MatVector: new () => any;
|
||||
[key: string]: any;
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/opencv",
|
||||
"version": "4.0.3",
|
||||
"version": "4.1.0",
|
||||
"description": "OpenCV-related helper library",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -23,26 +23,27 @@
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"author": "https://github.com/appium",
|
||||
"main": "./build/lib/index.js",
|
||||
"types": "./build/lib/index.d.ts",
|
||||
"directories": {
|
||||
"lib": "lib"
|
||||
},
|
||||
"files": [
|
||||
"index.js",
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
"test:e2e": "mocha --exit --timeout 20s --slow 10s \"./test/e2e/**/*.spec.js\"",
|
||||
"test:smoke": "node ./index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
"test:e2e": "mocha --exit --timeout 20s --slow 10s \"./test/e2e/**/*.spec.*ts\"",
|
||||
"test:smoke": "node ./build/lib/index.js",
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.*ts\""
|
||||
},
|
||||
"dependencies": {
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"opencv-bindings": "4.5.5",
|
||||
"sharp": "0.34.5"
|
||||
},
|
||||
|
||||
+60
-59
@@ -1,6 +1,10 @@
|
||||
import {getImagesMatches, getImagesSimilarity, getImageOccurrence} from '../../lib';
|
||||
import path from 'node:path';
|
||||
import {fs} from '@appium/support';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
const FIXTURES_ROOT = path.resolve(__dirname, 'images');
|
||||
|
||||
@@ -8,20 +12,15 @@ describe('OpenCV helpers', function () {
|
||||
// OpenCV needs several seconds for initialization
|
||||
this.timeout(120000);
|
||||
|
||||
let imgFixture = null;
|
||||
let fullImage = null;
|
||||
let partialImage = null;
|
||||
let originalImage = null;
|
||||
let changedImage = null;
|
||||
let rotatedImage = null;
|
||||
let numberImage = null;
|
||||
let imgFixture: Buffer | null = null;
|
||||
let fullImage: Buffer | null = null;
|
||||
let partialImage: Buffer | null = null;
|
||||
let originalImage: Buffer | null = null;
|
||||
let changedImage: Buffer | null = null;
|
||||
let rotatedImage: Buffer | null = null;
|
||||
let numberImage: Buffer | null = null;
|
||||
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
const chaiAsPromised = await import('chai-as-promised');
|
||||
chai.use(chaiAsPromised.default);
|
||||
chai.should();
|
||||
|
||||
const imagePath = path.resolve(FIXTURES_ROOT, 'full-image.b64');
|
||||
imgFixture = Buffer.from(await fs.readFile(imagePath, 'binary'), 'base64');
|
||||
fullImage = await fs.readFile(path.resolve(FIXTURES_ROOT, 'findwaldo.jpg'));
|
||||
@@ -34,112 +33,114 @@ describe('OpenCV helpers', function () {
|
||||
|
||||
describe('getImagesMatches', function () {
|
||||
it('should calculate the number of matches between two images', async function () {
|
||||
for (const detectorName of ['AKAZE', 'ORB']) {
|
||||
const {count, totalCount} = await getImagesMatches(fullImage, fullImage, {detectorName});
|
||||
count.should.be.above(0);
|
||||
totalCount.should.eql(count);
|
||||
for (const detectorName of ['AKAZE', 'ORB'] as const) {
|
||||
const {count, totalCount} = await getImagesMatches(fullImage!, fullImage!, {detectorName});
|
||||
expect(count).to.be.above(0);
|
||||
expect(totalCount).to.eql(count);
|
||||
}
|
||||
});
|
||||
|
||||
it('should visualize matches between two images', async function () {
|
||||
const {visualization} = await getImagesMatches(fullImage, fullImage, {
|
||||
const {visualization} = await getImagesMatches(fullImage!, fullImage!, {
|
||||
visualize: true,
|
||||
});
|
||||
visualization.should.not.be.empty;
|
||||
expect(visualization).to.not.be.empty;
|
||||
});
|
||||
|
||||
it('should visualize matches between two images and apply goodMatchesFactor', async function () {
|
||||
const {visualization, points1, rect1, points2, rect2} = await getImagesMatches(
|
||||
rotatedImage,
|
||||
originalImage,
|
||||
rotatedImage!,
|
||||
originalImage!,
|
||||
{
|
||||
visualize: true,
|
||||
matchFunc: 'BruteForceHamming',
|
||||
goodMatchesFactor: 40,
|
||||
}
|
||||
);
|
||||
visualization.should.not.be.empty;
|
||||
points1.length.should.be.above(4);
|
||||
rect1.x.should.be.above(0);
|
||||
rect1.y.should.be.above(0);
|
||||
rect1.width.should.be.above(0);
|
||||
rect1.height.should.be.above(0);
|
||||
points2.length.should.be.above(4);
|
||||
rect2.x.should.be.above(0);
|
||||
rect2.y.should.be.above(0);
|
||||
rect2.width.should.be.above(0);
|
||||
rect2.height.should.be.above(0);
|
||||
expect(visualization).to.not.be.empty;
|
||||
expect(points1.length).to.be.above(4);
|
||||
expect(rect1.x).to.be.above(0);
|
||||
expect(rect1.y).to.be.above(0);
|
||||
expect(rect1.width).to.be.above(0);
|
||||
expect(rect1.height).to.be.above(0);
|
||||
expect(points2.length).to.be.above(4);
|
||||
expect(rect2.x).to.be.above(0);
|
||||
expect(rect2.y).to.be.above(0);
|
||||
expect(rect2.width).to.be.above(0);
|
||||
expect(rect2.height).to.be.above(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getImagesSimilarity', function () {
|
||||
it('should calculate the similarity score between two images', async function () {
|
||||
const {score} = await getImagesSimilarity(imgFixture, imgFixture);
|
||||
score.should.be.above(0);
|
||||
const {score} = await getImagesSimilarity(imgFixture!, imgFixture!);
|
||||
expect(score).to.be.above(0);
|
||||
});
|
||||
|
||||
it('should visualize the similarity between two images', async function () {
|
||||
const {visualization} = await getImagesSimilarity(originalImage, changedImage, {
|
||||
const {visualization} = await getImagesSimilarity(originalImage!, changedImage!, {
|
||||
visualize: true,
|
||||
});
|
||||
visualization.should.not.be.empty;
|
||||
expect(visualization).to.not.be.empty;
|
||||
});
|
||||
});
|
||||
|
||||
describe('getImageOccurrence', function () {
|
||||
it('should calculate the partial image position in the full image', async function () {
|
||||
const {rect, score} = await getImageOccurrence(fullImage, partialImage);
|
||||
rect.x.should.be.above(0);
|
||||
rect.y.should.be.above(0);
|
||||
rect.width.should.be.above(0);
|
||||
rect.height.should.be.above(0);
|
||||
score.should.be.above(0);
|
||||
const {rect, score} = await getImageOccurrence(fullImage!, partialImage!);
|
||||
expect(rect.x).to.be.above(0);
|
||||
expect(rect.y).to.be.above(0);
|
||||
expect(rect.width).to.be.above(0);
|
||||
expect(rect.height).to.be.above(0);
|
||||
expect(score).to.be.above(0);
|
||||
});
|
||||
|
||||
it('should reject matches that fall below a threshold', async function () {
|
||||
await getImageOccurrence(fullImage, partialImage, {
|
||||
threshold: 1.0,
|
||||
}).should.eventually.be.rejectedWith(/threshold/);
|
||||
await expect(
|
||||
getImageOccurrence(fullImage!, partialImage!, {
|
||||
threshold: 1.0,
|
||||
})
|
||||
).to.eventually.be.rejectedWith(/threshold/);
|
||||
});
|
||||
|
||||
it('should visualize the partial image position in the full image', async function () {
|
||||
const {visualization} = await getImageOccurrence(fullImage, partialImage, {visualize: true});
|
||||
visualization.should.not.be.empty;
|
||||
const {visualization} = await getImageOccurrence(fullImage!, partialImage!, {visualize: true});
|
||||
expect(visualization).to.not.be.empty;
|
||||
});
|
||||
|
||||
describe('multiple', function () {
|
||||
it('should return matches in the full image', async function () {
|
||||
const {multiple} = await getImageOccurrence(originalImage, numberImage, {
|
||||
const {multiple} = await getImageOccurrence(originalImage!, numberImage!, {
|
||||
threshold: 0.8,
|
||||
multiple: true,
|
||||
});
|
||||
multiple.length.should.be.eq(3);
|
||||
expect(multiple).to.have.length(3);
|
||||
|
||||
for (const result of multiple) {
|
||||
result.rect.x.should.be.above(0);
|
||||
result.rect.y.should.be.above(0);
|
||||
result.rect.width.should.be.above(0);
|
||||
result.rect.height.should.be.above(0);
|
||||
result.score.should.be.above(0);
|
||||
for (const result of multiple!) {
|
||||
expect(result.rect.x).to.be.above(0);
|
||||
expect(result.rect.y).to.be.above(0);
|
||||
expect(result.rect.width).to.be.above(0);
|
||||
expect(result.rect.height).to.be.above(0);
|
||||
expect(result.score).to.be.above(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should reject matches that fall below a threshold', async function () {
|
||||
const {multiple} = await getImageOccurrence(originalImage, numberImage, {
|
||||
const {multiple} = await getImageOccurrence(originalImage!, numberImage!, {
|
||||
threshold: 1.0,
|
||||
multiple: true,
|
||||
});
|
||||
multiple.length.should.be.eq(1);
|
||||
expect(multiple).to.have.length(1);
|
||||
});
|
||||
|
||||
it('should visualize the partial image position in the full image', async function () {
|
||||
const {multiple} = await getImageOccurrence(originalImage, numberImage, {
|
||||
const {multiple} = await getImageOccurrence(originalImage!, numberImage!, {
|
||||
visualize: true,
|
||||
multiple: true,
|
||||
});
|
||||
|
||||
for (const result of multiple) {
|
||||
result.visualization.should.not.be.empty;
|
||||
for (const result of multiple!) {
|
||||
expect(result.visualization).to.not.be.empty;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,14 @@
|
||||
import {initOpenCv} from '../../lib';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
describe('OpenCV', function () {
|
||||
before(async function () {
|
||||
const chai = await import('chai');
|
||||
chai.should();
|
||||
});
|
||||
|
||||
it('should initialize opencv library', async function () {
|
||||
this.timeout('10s');
|
||||
await initOpenCv();
|
||||
const buildInfo = require('opencv-bindings').getBuildInformation();
|
||||
buildInfo.should.include('OpenCV');
|
||||
expect(buildInfo).to.include('OpenCV');
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,8 @@
|
||||
"extends": "@appium/tsconfig/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "build",
|
||||
"checkJs": true
|
||||
"checkJs": true,
|
||||
"types": ["mocha", "chai", "chai-as-promised"]
|
||||
},
|
||||
"include": ["lib"]
|
||||
"include": ["lib", "test"]
|
||||
}
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.0.5](https://github.com/appium/appium/compare/@appium/plugin-test-support@1.0.4...@appium/plugin-test-support@1.0.5) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/plugin-test-support
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [1.0.4](https://github.com/appium/appium/compare/@appium/plugin-test-support@1.0.3...@appium/plugin-test-support@1.0.4) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/plugin-test-support
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/plugin-test-support",
|
||||
"version": "1.0.4",
|
||||
"version": "1.0.5",
|
||||
"description": "Testing utilities for Appium plugins",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -33,7 +33,8 @@
|
||||
"index.js",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -41,10 +42,10 @@
|
||||
"test:unit": "exit 0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/types": "^1.1.2",
|
||||
"@appium/types": "^1.2.0",
|
||||
"get-port": "7.1.0",
|
||||
"log-symbols": "7.0.1",
|
||||
"teen_process": "4.0.7"
|
||||
"teen_process": "4.0.8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"appium": "^3.0.0-beta.0",
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [2.0.2](https://github.com/appium/appium/compare/@appium/relaxed-caps-plugin@2.0.1...@appium/relaxed-caps-plugin@2.0.2) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/relaxed-caps-plugin
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [2.0.1](https://github.com/appium/appium/compare/@appium/relaxed-caps-plugin@2.0.0...@appium/relaxed-caps-plugin@2.0.1) (2025-10-08)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/relaxed-caps-plugin",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.2",
|
||||
"description": "An Appium plugin that loosens requirements for vendor prefixes on caps",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -32,7 +32,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -40,7 +41,7 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"lodash": "4.17.21"
|
||||
"lodash": "4.17.23"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"appium": "^3.0.0-beta.0"
|
||||
|
||||
@@ -3,6 +3,16 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.0.1](https://github.com/appium/appium/compare/@appium/schema@1.0.0...@appium/schema@1.0.1) (2026-01-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
* **scripts:** properly import log-symbols in scripts ([#21819](https://github.com/appium/appium/issues/21819)) ([e9a6e54](https://github.com/appium/appium/commit/e9a6e5425cfd7f148ac2258a69884b4c3f2dbce3))
|
||||
|
||||
|
||||
|
||||
## [1.0.0](https://github.com/appium/appium/compare/@appium/schema@1.0.0-rc.1...@appium/schema@1.0.0) (2025-08-18)
|
||||
|
||||
**Note:** Version bump only for package @appium/schema
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/schema",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"description": "Appium Configuration Schema",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -29,7 +29,8 @@
|
||||
"lib",
|
||||
"index.js",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "node ./scripts/generate-schema-json.js",
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.0.5](https://github.com/appium/appium/compare/@appium/storage-plugin@1.0.4...@appium/storage-plugin@1.0.5) (2026-01-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [1.0.4](https://github.com/appium/appium/compare/@appium/storage-plugin@1.0.3...@appium/storage-plugin@1.0.4) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/storage-plugin
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/storage-plugin",
|
||||
"version": "1.0.4",
|
||||
"version": "1.0.5",
|
||||
"description": "Server-side storage plugin for Appium",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -32,7 +32,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -41,11 +42,11 @@
|
||||
"test:e2e": "mocha --timeout 20s --slow 10s \"./test/e2e/**/*.spec.*js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/support": "^7.0.5",
|
||||
"async-lock": "1.4.1",
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21",
|
||||
"lru-cache": "11.2.4",
|
||||
"lodash": "4.17.23",
|
||||
"lru-cache": "11.2.5",
|
||||
"rimraf": "6.1.2",
|
||||
"ws": "8.19.0"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.0.1](https://github.com/appium/appium/compare/@appium/strongbox@1.0.0...@appium/strongbox@1.0.1) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/strongbox
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [1.0.0](https://github.com/appium/appium/compare/@appium/strongbox@1.0.0-rc.1...@appium/strongbox@1.0.0) (2025-08-18)
|
||||
|
||||
**Note:** Version bump only for package @appium/strongbox
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/strongbox",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"description": "Persistent storage for Appium extensions",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -41,7 +41,7 @@
|
||||
"test:e2e": "mocha --exit -t 20s \"test/e2e/**/*.e2e.spec.ts\""
|
||||
},
|
||||
"dependencies": {
|
||||
"env-paths": "3.0.0",
|
||||
"env-paths": "4.0.0",
|
||||
"slugify": "1.6.6"
|
||||
},
|
||||
"engines": {
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [7.0.5](https://github.com/appium/appium/compare/@appium/support@7.0.4...@appium/support@7.0.5) (2026-01-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [7.0.4](https://github.com/appium/appium/compare/@appium/support@7.0.3...@appium/support@7.0.4) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/support
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/support",
|
||||
"version": "7.0.4",
|
||||
"version": "7.0.5",
|
||||
"description": "Support libs used across Appium packages",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -32,7 +32,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -41,12 +42,12 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/logger": "^2.0.3",
|
||||
"@appium/logger": "^2.0.4",
|
||||
"@appium/tsconfig": "^1.1.1",
|
||||
"@appium/types": "^1.1.2",
|
||||
"@appium/types": "^1.2.0",
|
||||
"@colors/colors": "1.6.0",
|
||||
"archiver": "7.0.1",
|
||||
"axios": "1.13.2",
|
||||
"axios": "1.13.3",
|
||||
"base64-stream": "1.0.0",
|
||||
"bluebird": "3.7.2",
|
||||
"bplist-creator": "0.1.1",
|
||||
@@ -57,7 +58,7 @@
|
||||
"jsftp": "2.1.3",
|
||||
"klaw": "4.1.0",
|
||||
"lockfile": "1.0.4",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"log-symbols": "7.0.1",
|
||||
"moment": "2.30.1",
|
||||
"ncp": "2.0.0",
|
||||
@@ -70,7 +71,7 @@
|
||||
"semver": "7.7.3",
|
||||
"shell-quote": "1.8.3",
|
||||
"supports-color": "10.2.2",
|
||||
"teen_process": "4.0.7",
|
||||
"teen_process": "4.0.8",
|
||||
"type-fest": "5.4.1",
|
||||
"uuid": "13.0.0",
|
||||
"which": "6.0.0",
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [4.0.5](https://github.com/appium/appium/compare/@appium/test-support@4.0.4...@appium/test-support@4.0.5) (2026-01-26)
|
||||
|
||||
**Note:** Version bump only for package @appium/test-support
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## [4.0.4](https://github.com/appium/appium/compare/@appium/test-support@4.0.3...@appium/test-support@4.0.4) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/test-support
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/test-support",
|
||||
"version": "4.0.4",
|
||||
"version": "4.0.5",
|
||||
"description": "A collection of test utilities used across Appium packages",
|
||||
"deprecated": "This module is deprecated. Use sinon directly instead (e.g., sinon.createSandbox() with Mocha hooks).",
|
||||
"keywords": [
|
||||
@@ -38,7 +38,8 @@
|
||||
"lib",
|
||||
"build",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -46,10 +47,10 @@
|
||||
"test:unit": "mocha \"./test/unit/**/*.spec.js\""
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/support": "^7.0.4",
|
||||
"@appium/support": "^7.0.5",
|
||||
"@colors/colors": "1.6.0",
|
||||
"bluebird": "3.7.2",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"loud-rejection": "2.2.0",
|
||||
"sinon": "21.0.1"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,21 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [1.2.0](https://github.com/appium/appium/compare/@appium/types@1.1.2...@appium/types@1.2.0) (2026-01-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **base-driver:** add ability to supply custom headers for proxy ([#21790](https://github.com/appium/appium/issues/21790)) ([a4b47c0](https://github.com/appium/appium/commit/a4b47c02c6e72381fa9d191d2a0bced106999705))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
* **scripts:** properly import log-symbols in scripts ([#21819](https://github.com/appium/appium/issues/21819)) ([e9a6e54](https://github.com/appium/appium/commit/e9a6e5425cfd7f148ac2258a69884b4c3f2dbce3))
|
||||
|
||||
|
||||
|
||||
## [1.1.2](https://github.com/appium/appium/compare/@appium/types@1.1.1...@appium/types@1.1.2) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/types
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/types",
|
||||
"version": "1.1.2",
|
||||
"version": "1.2.0",
|
||||
"description": "Various type declarations used across Appium",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -29,7 +29,8 @@
|
||||
"lib",
|
||||
"index.js",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "node ./scripts/generate-schema-types.js",
|
||||
@@ -38,8 +39,8 @@
|
||||
"test:types": "tsd"
|
||||
},
|
||||
"dependencies": {
|
||||
"@appium/logger": "^2.0.3",
|
||||
"@appium/schema": "^1.0.0",
|
||||
"@appium/logger": "^2.0.4",
|
||||
"@appium/schema": "^1.0.1",
|
||||
"@appium/tsconfig": "^1.1.1",
|
||||
"type-fest": "5.4.1"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
All notable changes to this project will be documented in this file.
|
||||
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
||||
|
||||
## [2.0.4](https://github.com/appium/appium/compare/@appium/universal-xml-plugin@2.0.3...@appium/universal-xml-plugin@2.0.4) (2026-01-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply npm run lint:fix ([#21867](https://github.com/appium/appium/issues/21867)) ([5e28714](https://github.com/appium/appium/commit/5e28714442cf59ee35b085e01e82c3f5483891fd))
|
||||
|
||||
|
||||
|
||||
## [2.0.3](https://github.com/appium/appium/compare/@appium/universal-xml-plugin@2.0.2...@appium/universal-xml-plugin@2.0.3) (2025-12-04)
|
||||
|
||||
**Note:** Version bump only for package @appium/universal-xml-plugin
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@appium/universal-xml-plugin",
|
||||
"version": "2.0.3",
|
||||
"version": "2.0.4",
|
||||
"description": "Appium plugin for making XML source and XPath queries the same across iOS and Android",
|
||||
"keywords": [
|
||||
"automation",
|
||||
@@ -29,7 +29,8 @@
|
||||
"build",
|
||||
"lib",
|
||||
"tsconfig.json",
|
||||
"!build/tsconfig.tsbuildinfo"
|
||||
"!build/tsconfig.tsbuildinfo",
|
||||
"!build/test"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "npm run test:unit",
|
||||
@@ -39,7 +40,7 @@
|
||||
"dependencies": {
|
||||
"@xmldom/xmldom": "0.9.8",
|
||||
"fast-xml-parser": "5.3.3",
|
||||
"lodash": "4.17.21",
|
||||
"lodash": "4.17.23",
|
||||
"xpath": "0.0.34"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
Reference in New Issue
Block a user