Add touch and multitouch actions for Android

This commit is contained in:
Isaac Murchie
2014-04-07 12:13:16 -07:00
parent 782bd87142
commit 0df0d2b2dc
9 changed files with 794 additions and 14 deletions
+233 -4
View File
@@ -997,12 +997,241 @@ androidController.unpackApp = function (req, cb) {
deviceCommon.unpackApp(req, '.apk', cb);
};
androidController.performTouch = function (gestures, cb) {
cb(new NotYetImplementedError(), null);
androidController.tap = function (elementId, x, y, count, cb) {
if (typeof x === "undefined" || x === null) x = 0;
if (typeof y === "undefined" || y === null) y = 0;
if (typeof count === "undefined" || count === null) count = 1;
var i = 0;
var opts = {};
var loop = function (err, res) {
if (err) return cb(err);
if (i++ >= count) return cb(err, res);
this.proxy(opts, loop);
}.bind(this);
if (x !== 0 || y !== 0) {
this.proxy(["element:getLocation", {elementId: elementId}], function (err, res) {
if (err) return cb(err);
var value = res.value;
x += parseInt(value.x);
y += parseInt(value.y);
opts = ["click", {x: x, y: y}];
loop();
}.bind(this));
} else {
opts = ["element:click", {elementId: elementId}];
loop();
}
};
androidController.performMultiAction = function (actions, cb) {
cb(new NotYetImplementedError(), null);
androidController.doTouchAction = function (action, opts, cb) {
switch (action) {
case 'tap':
return this.tap(opts.element, opts.x, opts.y, opts.count, cb);
case 'press':
return this.touchDown(opts.element, opts.x, opts.y, cb);
case 'release':
return this.touchUp(opts.element, opts.x, opts.y, cb);
case 'moveTo':
return this.touchMove(opts.element, opts.x, opts.y, cb);
case 'wait':
return setTimeout(cb, opts.ms);
case 'longPress':
if (typeof opts.duration === 'undefined' || !opts.duration) {
opts.duration = 1000;
}
return this.touchLongClick(opts.element, opts.x, opts.y, opts.duration, cb);
case 'cancel':
// TODO: clarify behavior of 'cancel' action and fix this
logger.warn("Cancel action currently has no effect");
break;
default:
return cb("unknown action '" + action + "'");
}
};
// drag is *not* press-move-release, so we need to translate
// drag works fine for scroll, as well
var doTouchDrag = function (gestures, cb) {
var longPress = gestures[0];
var elementId = longPress.options.element;
this.getLocation(elementId, function (err, res) {
if (err) return cb(err);
var startX = res.value.x;
var startY = res.value.y;
var moveTo = gestures[1];
var destElId = moveTo.options.element;
var endX = moveTo.options.x;
var endY = moveTo.options.y;
if (typeof destElId !== 'undefined' && destElId) {
this.getLocation(destElId, function (err, res) {
if (err) return cb(err);
endX = res.value.x + (endX || 0);
endY = res.value.y + (endY || 0);
return this.drag(startX, startY, endX, endY, 1, 1, elementId, destElId, cb);
}.bind(this));
} else {
return this.drag(startX, startY, endX, endY, 1, 1, elementId, destElId, cb);
}
}.bind(this));
return;
};
androidController.performTouch = function (gestures, cb) {
// some things are special
var actions = _.pluck(gestures, "action");
if (actions[0] === 'longPress' && actions[1] === 'moveTo' && actions[2] === 'release') {
return doTouchDrag.apply(this, [gestures, cb]);
} else if ((actions[actions.length - 2] === 'tap' ||
actions[actions.length - 2] === 'longPress') && actions[actions.length - 1] === 'release') {
// the `longPress` and `tap` methods release on their own
gestures.pop();
}
var cycleThroughGestures = function (err, res) {
if (err) return cb(err);
var gesture = gestures.shift();
if (typeof gesture === "undefined") return cb(null, res);
this.doTouchAction(gesture.action, gesture.options, cycleThroughGestures);
}.bind(this);
cycleThroughGestures();
};
androidController.parseTouch = function (gestures, cb) {
if (_.last(gestures).action === 'release') {
gestures.pop();
}
var needsPoint = function (action) {
return _.contains(['press', 'moveTo', 'tap', 'longPress'], action);
};
var touchStateObjects = [];
async.eachSeries(gestures, function (gesture, done) {
var tapPoint = false;
if (needsPoint(gesture.action)) { // press, longPress, moveTo and tap all need a position
var elementId = gesture.options.element;
if (elementId) {
this.getLocation(elementId, function (err, res) {
if (err) return done(err); // short circuit and quit
var pos = { x: res.value.x, y: res.value.y };
this.getSize(elementId, function (err, res) {
if (err) return done(err);
var size = {w: res.value.width, h: res.value.height};
if (gesture.options.x || gesture.options.y) {
tapPoint = {
offset: false,
x: pos.x + (gesture.options.x || 0),
y: pos.y + (gesture.options.y || 0)
};
} else {
tapPoint = {
offset: false,
x: pos.x + (size.w / 2),
y: pos.y + (size.h / 2)
};
}
var touchStateObject = {
timeOffset: 0.005,
touch: tapPoint
};
touchStateObjects.push(touchStateObject);
done();
});
}.bind(this));
} else {
// expects absolute coordinates, so we need to save these as offsets
// and then translate when everything is done
tapPoint = {
offset: true,
x: (gesture.options.x || 0),
y: (gesture.options.y || 0)
};
touchStateObject = {
timeOffset: 0.005,
touch: tapPoint
};
touchStateObjects.push(touchStateObject);
done();
}
} else {
// in this case we need the previous entry's tap point
tapPoint = false; // temporary marker
var offset = 0.005;
if (gesture.action === 'wait') {
if (typeof gesture.options.ms !== 'undefined' || gesture.options.ms !== null) {
offset = (parseInt(gesture.options.ms) / 1000);
}
}
var touchStateObject = {
timeOffset: offset,
touch: tapPoint
};
touchStateObjects.push(touchStateObject);
done();
}
}.bind(this), function (err) {
if (err) return cb(err);
// we need to change the time (which is now an offset)
// and the position (which may be an offset)
var prevPos = null,
time = 0;
_.each(touchStateObjects, function (state, index) {
if (state.touch === false) {
// if we have no position (this happens with `wait`) we need the previous one
state.touch = prevPos;
} else if (state.touch.offset && prevPos) {
// the current position is an offset
state.touch.x += prevPos.x;
state.touch.y += prevPos.y;
}
delete state.touch.offset;
prevPos = state.touch;
var timeOffset = state.timeOffset;
time += timeOffset;
state.time = helpers.truncateDecimals(time, 3);
delete state.timeOffset;
});
cb(null, touchStateObjects);
});
};
androidController.performMultiAction = function (elementId, actions, cb) {
var states = [];
async.eachSeries(actions, function (action, done) {
this.parseTouch(action, function (err, val) {
if (err) return done(err);
states.push(val);
done();
}.bind(this));
}.bind(this), function (err) {
if (err) return cb(err);
var opts = {
elementId: elementId,
actions: states
};
return this.proxy(["element:performMultiPointerGesture", opts], cb);
}.bind(this));
};
module.exports = androidController;
@@ -16,6 +16,7 @@ import io.appium.android.bootstrap.handler.GetName;
import io.appium.android.bootstrap.handler.GetSize;
import io.appium.android.bootstrap.handler.GetStrings;
import io.appium.android.bootstrap.handler.GetText;
import io.appium.android.bootstrap.handler.MultiPointerGesture;
import io.appium.android.bootstrap.handler.Orientation;
import io.appium.android.bootstrap.handler.Pinch;
import io.appium.android.bootstrap.handler.PressBack;
@@ -38,7 +39,7 @@ import org.json.JSONException;
/**
* Command execution dispatch class. This class relays commands to the various
* handlers.
*
*
*/
class AndroidCommandExecutor {
@@ -75,11 +76,12 @@ class AndroidCommandExecutor {
new EnableCompressedLayoutHeirarchy());
map.put("getStrings", new GetStrings());
map.put("getDataDir", new GetDataDir());
map.put("performMultiPointerGesture", new MultiPointerGesture());
}
/**
* Gets the handler out of the map, and executes the command.
*
*
* @param command
* The {@link AndroidCommand}
* @return {@link AndroidCommandResult}
@@ -10,9 +10,13 @@ import com.android.uiautomator.core.UiObject;
import com.android.uiautomator.core.UiObjectNotFoundException;
import com.android.uiautomator.core.UiSelector;
import android.view.MotionEvent.PointerCoords;
import java.lang.reflect.Method;
/**
* Proxy class for UiObject.
*
*
*/
public class AndroidElement {
@@ -231,4 +235,23 @@ public class AndroidElement {
return el.setText(text);
}
public boolean performMultiPointerGesture(PointerCoords[] ...touches) {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
// The compile-time SDK expects the wrong arguments, but the runtime
// version in the emulator is correct. So we cannot do:
// `return el.performMultiPointerGesture(touches);`
// Instead we need to use Reflection to do it all at runtime.
Method method = this.el.getClass().getMethod("performMultiPointerGesture", PointerCoords[][].class);
Boolean rt = (Boolean)method.invoke(this.el, (Object)touches);
return rt.booleanValue();
} else {
Logger.error("Device does not support API < 18!");
return false;
}
} catch (final Exception e) {
Logger.error("Exception: " + e + " (" + e.getMessage() + ")");
return false;
}
}
}
@@ -2,7 +2,7 @@ package io.appium.android.bootstrap;
/**
* Log to standard out so that the Appium framework can pick it up.
*
*
*/
public class Logger {
@@ -14,10 +14,10 @@ public class Logger {
}
public static void error(final String msg) {
System.out.println(Logger.prefix + " [debug] " + msg + Logger.suffix);
System.out.println(Logger.prefix + " [error] " + msg + Logger.suffix);
}
public static void info(final String msg) {
System.out.println(Logger.prefix + " [info] " + msg + Logger.suffix);
}
}
}
@@ -0,0 +1,123 @@
package io.appium.android.bootstrap.handler;
import android.view.MotionEvent.PointerCoords;
import com.android.uiautomator.core.UiObject;
import com.android.uiautomator.core.UiObjectNotFoundException;
import com.android.uiautomator.core.UiScrollable;
import com.android.uiautomator.core.UiSelector;
import org.json.JSONException;
import org.json.JSONArray;
import org.json.JSONObject;
import io.appium.android.bootstrap.AndroidCommand;
import io.appium.android.bootstrap.AndroidCommandResult;
import io.appium.android.bootstrap.AndroidElement;
import io.appium.android.bootstrap.CommandHandler;
import io.appium.android.bootstrap.Logger;
import io.appium.android.bootstrap.WDStatus;
import io.appium.android.bootstrap.exceptions.ElementNotInHashException;
import java.lang.reflect.Method;
public class MultiPointerGesture extends CommandHandler {
@Override
public AndroidCommandResult execute(final AndroidCommand command)
throws JSONException {
if (!command.isElementCommand()) {
return getErrorResult("A gesturable view is required for this command.");
}
try {
JSONArray actions = (org.json.JSONArray)command.params().get("actions");
double time = computeLongestTime(actions);
PointerCoords[][] pcs = new PointerCoords[actions.length()][];
for (int i = 0; i < actions.length(); i++) {
JSONArray gestures = actions.getJSONArray(i);
pcs[i] = gesturesToPointerCoords(time, gestures);
}
final AndroidElement el = command.getElement();
if (el.performMultiPointerGesture(pcs)) {
return getSuccessResult("OK");
} else {
return getErrorResult("Unable to perform multi pointer gesture");
}
} catch (final ElementNotInHashException e) {
return new AndroidCommandResult(WDStatus.NO_SUCH_ELEMENT, e.getMessage());
} catch (final Exception e) {
Logger.debug("Exception: " + e);
return new AndroidCommandResult(WDStatus.UNKNOWN_ERROR, e.getMessage());
}
}
private PointerCoords[] gesturesToPointerCoords(double maxTime, JSONArray gestures) throws Exception {
// gestures, e.g.:
// [
// {"touch":{"y":529.5,"x":120},"time":0.2},
// {"touch":{"y":529.5,"x":130},"time":0.4},
// {"touch":{"y":454.5,"x":140},"time":0.6},
// {"touch":{"y":304.5,"x":150},"time":0.8}
// ]
// From the docs:
// "Steps are injected about 5 milliseconds apart, so 100 steps may take
// around 0.5 seconds to complete."
int steps = (int)(maxTime * 200) + 2;
PointerCoords[] pc = new PointerCoords[steps];
int i = 1;
JSONObject current = gestures.getJSONObject(0);
double currentTime = current.getDouble("time");
double runningTime = 0.0;
int gesturesLength = gestures.length();
for (int j = 0; j < steps; j++) {
if (runningTime > currentTime && i < gesturesLength) {
current = gestures.getJSONObject(i++);
currentTime = current.getDouble("time");
}
pc[j] = createPointerCoords(current);
runningTime += 0.005;
}
return pc;
}
private PointerCoords createPointerCoords(JSONObject obj) throws JSONException {
JSONObject o = obj.getJSONObject("touch");
int x = o.getInt("x");
int y = o.getInt("y");
PointerCoords p = new PointerCoords();
p.size = 1;
p.pressure = 1;
p.x = x;
p.y = y;
return p;
}
private double computeLongestTime(JSONArray actions) throws JSONException {
double max = 0.0;
for (int i = 0; i < actions.length(); i++) {
JSONArray gestures = actions.getJSONArray(i);
double endTime = gestures.getJSONObject(gestures.length()-1).getDouble("time");
if (endTime > max) {
max = endTime;
}
}
return max;
}
}
+1 -2
View File
@@ -1960,7 +1960,6 @@ iOSController.parseTouch = function (gestures, cb) {
return _.contains(['press', 'moveTo', 'tap', 'longPress'], action);
};
// _.each(gestures, function (gesture, index) {
var cycleThroughGestures = function () {
var gesture = gestures.shift();
if (typeof gesture === "undefined") {
@@ -2078,7 +2077,7 @@ var mergeStates = function (states) {
return merged;
};
iOSController.performMultiAction = function (actions, cb) {
iOSController.performMultiAction = function (elementId, actions, cb) {
var states = [];
var cycleThroughActions = function () {
var action = actions.shift();
+8
View File
@@ -623,3 +623,11 @@ exports.getAppiumConfig = function () {
exports.iosConfigured = function () {
return typeof exports.getAppiumConfig().ios !== "undefined";
};
exports.truncateDecimals = function (number, digits) {
var multiplier = Math.pow(10, digits),
adjustedNum = number * multiplier,
truncatedNum = Math[adjustedNum < 0 ? 'ceil' : 'floor'](adjustedNum);
return truncatedNum / multiplier;
};
+3 -2
View File
@@ -256,9 +256,10 @@ exports.performTouch = function (req, res) {
};
exports.performMultiAction = function (req, res) {
var actions = req.body;
var elementId = req.body.elementId;
var actions = req.body.actions;
req.device.performMultiAction(actions, getResponseHandler(req, res));
req.device.performMultiAction(elementId, actions, getResponseHandler(req, res));
};
exports.doClick = function (req, res) {
@@ -0,0 +1,395 @@
"use strict";
var env = require('../../../helpers/env')
, setup = require("../../common/setup-base")
, desired = require("./desired")
, androidReset = require('../../../helpers/reset').androidReset
, Q = require("q")
, _ = require("underscore")
, wd = require("wd")
, TouchAction = wd.TouchAction
, MultiAction = wd.MultiAction;
describe("apidemo - gestures -", function () {
var driver;
setup(this, desired).then(function (d) { driver = d; });
if (env.FAST_TESTS) {
beforeEach(function (done) {
androidReset('com.example.android.apis', '.ApiDemos').nodeify(done);
});
}
describe('tap visible element', function () {
it('should tap an element', function (done) {
driver.elementByName("Animation")
.then(function (el) {
var action = new TouchAction(el);
action.tap().perform();
})
.elementsByTagName("text").then(function (els) { return els[1]; })
.text().should.become("Bouncing Balls")
.nodeify(done);
});
it('should tap an element from an offset', function (done) {
driver.elementByName("Animation")
.then(function (el) {
var action = new TouchAction(el);
return action.tap({ x: 1, y: 1 }).perform();
})
.elementsByTagName("text").then(function (els) { return els[1]; })
.text().should.become("Bouncing Balls")
.nodeify(done);
});
it('should tap an element twice', function (done) {
driver
.elementByName("Text")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.elementByName("LogTextBox")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.elementByName("Add")
.then(function (el) {
var action = new TouchAction(el);
return action.tap({ count: 2 }).perform();
})
.elementsByTagName("text")
.then(function (els) {
return els[1];
})
.text()
.should.become("This is a test\nThis is a test\n")
.nodeify(done);
});
it('should tap an element from an offset twice', function (done) {
driver
.elementByName("Text")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.elementByName("LogTextBox")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.elementByName("Add")
.then(function (el) {
var action = new TouchAction(el);
return action.tap({ count: 2 }).perform();
})
.elementsByTagName("text")
.then(function (els) {
return els[1];
})
.text()
.should.become("This is a test\nThis is a test\n")
.nodeify(done);
});
});
describe('press', function () {
it('should work like `tap` when immediately released', function (done) {
driver.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press();
driver
.elementByName("Animation")
.then(function (el2) {
return action.moveTo({ element: el2.value.toString() }).release().perform();
});
})
.sleep(500)
.elementByName("Views")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("Expandable Lists")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("1. Custom Adapter")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("People Names") //.should.eventually.exist
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.elementByName("Sample menu").should.be.rejected
.nodeify(done);
});
it('should work like `longPress` when released after a pause', function (done) {
driver.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press();
driver
.elementByName("Animation")
.then(function (el2) {
return action.moveTo({ element: el2.value.toString() }).release().perform();
});
})
.sleep(500)
.elementByName("Views")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("Expandable Lists")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("1. Custom Adapter")
.then(function (el) {
var action = new TouchAction(el);
return action.press().release().perform();
})
.sleep(500)
.elementByName("People Names") //.should.eventually.exist
.then(function (el) {
var action = new TouchAction(el);
return action.press().wait(1000).release().perform();
})
.elementByName("Sample menu").should.eventually.exist
.nodeify(done);
});
});
describe('longPress', function () {
it('should open a context menu', function (done) {
driver.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press();
driver
.elementByName("Animation")
.then(function (el2) {
return action.moveTo({ element: el2.value.toString() }).release().perform();
});
})
.sleep(500)
.elementByName("Views")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.sleep(500)
.elementByName("Expandable Lists")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.sleep(500)
.elementByName("1. Custom Adapter")
.then(function (el) {
var action = new TouchAction(el);
return action.tap().perform();
})
.sleep(500)
.elementByName("People Names")
.then(function (el) {
var action = new TouchAction(el);
return action.longPress().release().perform();
})
.elementByName("Sample menu").should.eventually.exist
.nodeify(done);
});
});
describe('swipe', function () {
it('should be possible with press/moveTo/release', function (done) {
driver.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press();
driver
.elementByName("Animation")
.then(function (el2) {
return action.moveTo({ element: el2.value.toString() }).release().perform();
});
})
.sleep(500)
.elementByName("Views").should.eventually.exist
.nodeify(done);
});
it('should be possible with press/moveTo/release and pixel offset', function (done) {
driver.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press().moveTo({ x: 0, y: -400 }).release().perform();
})
.sleep(500)
.elementByName("Views").should.eventually.exist
.nodeify(done);
});
});
describe('drag', function () {
it('should drag by pixels @skip-android-all', function (done) {
driver
.elementByName("Content")
.then(function (el) {
var action = new TouchAction(el);
action.press();
driver
.elementByName("Animation")
.then(function (el2) {
return action.moveTo({ element: el2.value.toString() }).release().perform();
});
})
.sleep(500)
.elementByName("Views")
.then(function (el) {
var action = new TouchAction(el);
action.tap().perform();
})
.sleep(500)
.elementByName("Drag and Drop")
.then(function (el) {
var action = new TouchAction(el);
action.tap().perform();
})
.sleep(500)
.elementById("com.example.android.apis:id/drag_dot_3")
.then(function (dd3) {
return driver
.elementById("com.example.android.apis:id/drag_dot_2")
.then(function (dd2) {
var action = new TouchAction(dd3);
return action.longPress().moveTo({ element: dd2.value.toString() }).release().perform();
});
})
.sleep(1500)
.elementById("com.example.android.apis:id/drag_result_text").text()
.should.become("Dropped!")
.sleep(15000)
.nodeify(done);
});
});
describe('multi actions', function () {
it('should scroll two different lists', function (done) {
var scrollOpts = {};
driver
.elementByTagName("listView")
.then(function (el) {
scrollOpts = {
element: el.value
, text: 'Views'
};
return driver.execute("mobile: scrollTo", [scrollOpts]);
}).elementByXPath("//text[@value='Views']")
.then(function (el) {
return new TouchAction(el).tap().perform();
})
.then(function () {
scrollOpts.text = 'Splitting Touches across Views';
return driver.execute("mobile: scrollTo", [scrollOpts]);
}).elementByXPath("//text[@value='Splitting Touches across Views']")
.then(function (el) {
return new TouchAction(el).tap().perform();
})
.elementsByTagName("listView")
.then(function (els) {
// scroll slowly on the left
var a1 = new TouchAction(els[0]);
a1
.press()
.moveTo({ x: 10, y: 0 })
.moveTo({ x: 10, y: -75 })
.moveTo({ x: 10, y: -150 })
.release();
// scross quickly on the right
var a2 = new TouchAction(els[1]);
a2
.press()
.moveTo({ x: 10, y: 0 })
.moveTo({ x: 10, y: -300 })
.moveTo({ x: 10, y: -600 })
.release();
var ma = new MultiAction(els[0]);
ma.add(a1, a2);
ma.perform();
})
.sleep(15000)
.nodeify(done);
});
it('should scroll two different lists with waits', function (done) {
var scrollOpts = {};
driver
.elementByTagName("listView")
.then(function (el) {
scrollOpts = {
element: el.value
, text: 'Views'
};
return driver.execute("mobile: scrollTo", [scrollOpts]);
}).elementByXPath("//text[@value='Views']")
.then(function (el) {
return new TouchAction(el).tap().perform();
})
.then(function () {
scrollOpts.text = 'Splitting Touches across Views';
return driver.execute("mobile: scrollTo", [scrollOpts]);
}).elementByXPath("//text[@value='Splitting Touches across Views']")
.then(function (el) {
return new TouchAction(el).tap().perform();
})
.elementsByTagName("listView")
.then(function (els) {
// scroll slowly on the left
var a1 = new TouchAction(els[0]);
a1
.press()
.moveTo({ x: 10, y: 0 })
.moveTo({ x: 10, y: -75 })
.wait(1000)
.moveTo({ x: 10, y: -350 })
.release();
// scross quickly on the right
var a2 = new TouchAction(els[1]);
a2
.press()
.moveTo({ x: 10, y: 100 })
.moveTo({ x: 10, y: -300 })
.wait(500)
.moveTo({ x: 10, y: -600 })
.release();
var ma = new MultiAction(els[0]);
ma.add(a1, a2);
ma.perform();
})
.sleep(15000)
.nodeify(done);
});
});
});