import _ from 'lodash'; import { errors } from '../..'; import LRU from 'lru-cache'; import { IMAGE_ELEMENT_PREFIX } from '../constants'; import log from './logger'; import { util } from 'appium-support'; const MAX_CACHE_SIZE = 1024 * 1024 * 40; // 40mb const TAP_DURATION_MS = 125; const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions'; const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions'; const IMAGE_TAP_STRATEGIES = [ IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C ]; const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0; /** * @typedef {Object} Rect * @property {int} x - x-coordinate of top-left corner * @property {int} y - y-coordinate of top-left corner * @property {int} width - width of rect * @property {int} height - height of rect */ /** * @typedef {Object} Dimension * @property {int} width - width of rect * @property {int} height - height of rect */ /** * @typedef {Object} Position * @property {int} x - x coordinate * @property {int} y - y coordinate */ /** * Representation of an "image element", which is simply a set of coordinates * and methods that can be used on that set of coordinates via the driver */ class ImageElement { /** * @param {string} b64Template - the base64-encoded image which was used to * find this ImageElement * @param {Rect} rect - bounds of matched image element * @param {number} score The similarity score as a float number in range [0.0, 1.0]. * 1.0 is the highest score (means both images are totally equal). * @param {?string} b64Result - the base64-encoded image which has matched marks. * Defaults to null. * * @returns {ImageElement} */ constructor (b64Template, rect, score, b64Result = null) { this.template = b64Template; this.rect = rect; this.id = `${IMAGE_ELEMENT_PREFIX}${util.uuidV4()}`; this.b64MatchedImage = b64Result; this.score = score; } /** * @returns {Dimension} - dimension of element */ get size () { return {width: this.rect.width, height: this.rect.height}; } /** * @returns {Position} - coordinates of top-left corner of element */ get location () { return {x: this.rect.x, y: this.rect.y}; } /** * @returns {Position} - coordinates of center of element */ get center () { return { x: this.rect.x + this.rect.width / 2, y: this.rect.y + this.rect.height / 2, }; } /** * @returns {?string} - the base64-encoded image which has matched marks */ get matchedImage () { return this.b64MatchedImage; } /** * @param {string} protocolKey - the protocol-specific JSON key for * a WebElement * * @returns {WebElement} - this image element as a WebElement */ asElement (protocolKey) { return {[protocolKey]: this.id}; } /** * @param {ImageElement} other - an ImageElement to compare with this one * * @returns {boolean} - whether the other element and this one have the same * properties */ equals (other) { return this.rect.x === other.rect.x && this.rect.y === other.rect.y && this.rect.width === other.rect.width && this.rect.height === other.rect.height; } /** * Use a driver to tap the screen at the center of this ImageElement's * position * * @param {BaseDriver} driver - driver for calling actions with */ async click (driver) { // before we click we need to make sure the element is actually still there // where we expect it to be let newImgEl; const { autoUpdateImageElementPosition: updatePos, checkForImageElementStaleness, imageElementTapStrategy, } = driver.settings.getSettings(); // validate tap strategy if (!IMAGE_TAP_STRATEGIES.includes(imageElementTapStrategy)) { throw new Error(`Incorrect imageElementTapStrategy setting ` + `'${imageElementTapStrategy}'. Must be one of ` + JSON.stringify(IMAGE_TAP_STRATEGIES)); } if (checkForImageElementStaleness || updatePos) { log.info('Checking image element for staleness before clicking'); try { newImgEl = await driver.findByImage(this.template, { shouldCheckStaleness: true, // Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image // managed inside Appium after finidng image by template which managed by a user ignoreDefaultImageTemplateScale: true }); } catch (err) { throw new errors.StaleElementReferenceError(); } if (!this.equals(newImgEl)) { log.warn(`When trying to click on an image element, the image changed ` + `position from where it was originally found. It is now at ` + `${JSON.stringify(newImgEl.rect)} and was originally at ` + `${JSON.stringify(this.rect)}.`); if (updatePos) { log.warn('Click will proceed at new coordinates'); this.rect = _.clone(newImgEl.rect); } else { log.warn('Click will take place at original coordinates. If you ' + 'would like Appium to automatically click the new ' + "coordinates, set the 'autoUpdateImageElementPosition' " + 'setting to true'); } } } const {x, y} = this.center; log.info(`Will tap on image element at coordinate [${x}, ${y}]`); if (imageElementTapStrategy === IMAGE_EL_TAP_STRATEGY_W3C) { // set up a W3C action to click on the image by position log.info('Will tap using W3C actions'); const action = { type: 'pointer', id: 'mouse', parameters: {pointerType: 'touch'}, actions: [ {type: 'pointerMove', x, y, duration: 0}, {type: 'pointerDown', button: 0}, {type: 'pause', duration: TAP_DURATION_MS}, {type: 'pointerUp', button: 0}, ] }; // check if the driver has the appropriate performActions method if (driver.performActions) { return await driver.performActions([action]); } // if not, warn and fall back to the other method log.warn('Driver does not seem to implement W3C actions, falling back ' + 'to TouchActions'); } // if the w3c strategy was not requested, do the only other option (mjsonwp // touch actions) log.info('Will tap using MJSONWP TouchActions'); const action = { action: 'tap', options: {x, y} }; if (driver.performTouch) { return await driver.performTouch([action]); } throw new Error("Driver did not implement the 'performTouch' command. " + 'For drivers to support finding image elements, they ' + "should support 'performTouch' and 'performActions'"); } /** * Handle various Appium commands that involve an image element * * @param {BaseDriver} driver - the driver to use for commands * @param {string} cmd - the name of the driver command * @param {string} imgElId - the id of the ImageElement to work with * @param {Array} args - Rest of arguments for executeScripts * * @returns {Object} - the result of running a command */ static async execute (driver, cmd, imgElId, ...args) { if (!driver._imgElCache.has(imgElId)) { throw new errors.NoSuchElementError(); } const imgEl = driver._imgElCache.get(imgElId); switch (cmd) { case 'click': return await imgEl.click(driver); case 'elementDisplayed': return true; case 'getSize': return imgEl.size; case 'getLocation': case 'getLocationInView': return imgEl.location; case 'getElementRect': return imgEl.rect; case 'getAttribute': // /session/:sessionId/element/:elementId/attribute/:name // /session/:sessionId/element/:elementId/attribute/visual should retun the visual data // e.g. ["content-desc","appium-image-element-xxxxx","xxxxx"], ["visual","appium-image-element-xxxxx","xxxxx"] switch (args[0]) { case 'visual': return imgEl.matchedImage; case 'score': return imgEl.score; default: throw new errors.NotYetImplementedError(); } default: throw new errors.NotYetImplementedError(); } } } function makeImageElementCache (max = MAX_CACHE_SIZE) { return new LRU({ max, length: (el) => el.template.length, }); } function getImgElFromArgs (args) { for (let arg of args) { if (_.isString(arg) && arg.startsWith(IMAGE_ELEMENT_PREFIX)) { return arg; } } } export { ImageElement, getImgElFromArgs, makeImageElementCache, IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C, DEFAULT_TEMPLATE_IMAGE_SCALE };