tt/AutoJs6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: images.findImage; change images.findColor implementation to opencv

Browse Source
This commit is contained in:
hyb1996
2017-11-26 17:47:17 +08:00
parent 35955faba9
commit 328ab6efa7
25 changed files with 551 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
autojs/src/main/assets/modules/__http__.js
View File

@@ -37,7 +37,7 @@ module.exports = function(runtime, scope){
}
call.enqueue(new Callback({
onResponse: function(call, res){
callback(res);
callback(wrapResponse(res));
},
onFailure: function(call, ex){
callback(null, ex);
@@ -103,6 +103,7 @@ module.exports = function(runtime, scope){
function wrapResponse(res){
var r = {};
r.statusCode = res.code();
r.statusMessage = res.message();
var headers = res.headers();
r.headers = {};
for(var i = 0; i < headers.size(); i++){
@@ -112,6 +113,7 @@ module.exports = function(runtime, scope){
r.body.json = function(){
return JSON.parse(r.body.string());
}
r.body.contentType = r.body.contentType();
r.request = res.request();
r.url = r.request.url();
r.method = r.request.method();

94
autojs/src/main/assets/modules/__images__.js
View File

@@ -17,13 +17,15 @@ module.exports = function(__runtime__, scope){
images.captureScreen = rtImages.captureScreen.bind(rtImages);
images.read = rtImages.read.bind(rtImages);
images.saveImage = rtImages.saveImage.bind(rtImages);
images.pixel = rtImages.pixel;
images.detectsColor = function(img, color, x, y, threshold, algorithm){
color = parseColor(color);
algorithm = algorithm || "rgb";
algorithm = algorithm || "diff";
threshold = threshold || 16;
var colorDetector = getColorDetector(color, algorithm, threshold);
var pixel = images.pixel(img, x, y);
@@ -34,54 +36,80 @@ module.exports = function(__runtime__, scope){
color = parseColor(color);
options = options || {};
var region = options.region || [];
var x = region[0] || 0;
var y = region[1] || 0;
var width = region[2] || (img.getWidth() - x);
var height = region[3] || (img.getHeight() - y);
var threads = options.threads || 2;
if(options.similarity){
var threshold = parseInt(255 * (1 - options.similarity));
}else{
var threshold = options.threshold || 16;
}
algorithm = options.algorithm || "rgb";
var rect = new android.graphics.Rect(x, y, width + x, height + y);
var colorDetector = getColorDetector(color, algorithm, threshold);
return colorFinder.findColorConcurrently(img, colorDetector, rect, threads);
if(options.region){
return rtImages.toAndroidPoint(colorFinder.findColor(img, color, threshold, buildRegion(options.region, img)));
}else{
return rtImages.toAndroidPoint(colorFinder.findColor(img, color, threshold, null));
}
}
images.findColorInRegion = function(img, color, x, y, width, height, threads, algorithm, threshold){
images.findColorInRegion = function(img, color, x, y, width, height, threshold){
return findColor(img, color, {
region: [x, y, width, height],
algorithm: algorithm,
threshold: threshold,
threads: threads
threshold: threshold
});
}
images.findColorEquals = function(img, color, x, y, width, height, threads){
images.findColorEquals = function(img, color, x, y, width, height){
return findColor(img, color, {
region: [x, y, width, height],
algorithm: "equal",
threads: threads
threshold: 0
});
}
function getColorDetector(color, algorithm, threshold){
switch(algorithm){
case "rgb":
return new com.stardust.autojs.core.image.ColorDetector.RGBDistanceDetector(color, threshold);
case "equal":
return new com.stardust.autojs.core.image.ColorDetector.EqualityDetector(color);
case "diff":
return new com.stardust.autojs.core.image.ColorDetector.DifferenceDetector(color, threshold);
case "rgb+":
return new com.stardust.autojs.core.image.ColorDetector.WeightedRGBDistanceDetector(color, threshold);
case "hs":
return new com.stardust.autojs.core.image.ColorDetector.HSDistanceDetector(color, threshold);
}
throw new Error("Unknown algorithm: " + algorithm);
}
images.findColors = function(img, color, options){
color = parseColor(color);
options = options || {};
if(options.similarity){
var threshold = parseInt(255 * (1 - options.similarity));
}else{
var threshold = options.threshold || 16;
}
if(options.region){
return toPointArray(colorFinder.findAllColors(img, color, threshold, buildRegion(options.region, img)));
}else{
return toPointArray(colorFinder.findAllColors(img, color, threshold, null));
}
}
images.findImage = function(img, template, options){
options = options || {};
var threshold = options.threshold || 0.9;
var maxLevel = options.level || -1;
if(options.region){
return rtImages.findImage(img, template, threshold, buildRegion(options, img), maxLevel);
}else{
return rtImages.findImage(img, template, threshold, null, maxLevel);
}
}
images.findImageInRegion = function(img, template, x, y, width, height, threshold){
return images.findImage(img, template, {
region: [x, y, width, height],
threshold: threshold
});
}
function toPointArray(points){
var arr = [];
for(var i = 0; i < points.length; i++){
arr.push(rtImages.toAndroidPoint(points[i]));
}
return arr;
}
function buildRegion(region, img){
var x = region[0] || 0;
var y = region[1] || 0;
var width = region[2] || (img.getWidth() - x);
var height = region[3] || (img.getHeight() - y);
return new org.opencv.core.Rect(x, y, width, height);
}
function parseColor(color){
if(typeof(color) == 'string'){
@@ -94,7 +122,7 @@ module.exports = function(__runtime__, scope){
return color;
}
scope.__asGlobal__(images, ['requestScreenCapture', 'captureScreen', 'findColor', 'findColorInRegion', 'findColorEquals']);
scope.__asGlobal__(images, ['requestScreenCapture', 'captureScreen', 'findImage', 'findImageInRegion', 'findColor', 'findColorInRegion', 'findColorEquals']);
scope.colors = colors;

4
autojs/src/main/assets/modules/__util__.js
View File

@@ -24,8 +24,8 @@ exports.format = function(f) {
if (!isString(f)) {
var objects = [];
for (var i = 0; i < arguments.length; i++) {
var v = isJavaObject(arguments[i]) ? arguments[i].toString() :
arguments[i];
var v = (arguments[i] && isJavaObject(arguments[i])) ? arguments[i].toString() :
arguments[i];
objects.push(inspect(v));
}
return objects.join(' ');

199
autojs/src/main/java/com/stardust/autojs/core/image/ColorDetector.java
View File

@@ -1,199 +0,0 @@
package com.stardust.autojs.core.image;
import android.graphics.Color;
/**
* Created by Stardust on 2017/5/20.
*/
public interface ColorDetector {
boolean detectsColor(int red, int green, int blue);
abstract class AbstractColorDetector implements ColorDetector {
protected final int mColor;
protected final int mR, mG, mB;
public AbstractColorDetector(int color) {
mColor = color;
mR = Color.red(color);
mG = Color.green(color);
mB = Color.blue(color);
}
}
class EqualityDetector extends AbstractColorDetector {
public EqualityDetector(int color) {
super(color);
}
@Override
public boolean detectsColor(int red, int green, int blue) {
return mR == red && mG == green && mB == blue;
}
}
class DifferenceDetector extends AbstractColorDetector {
private final int mThreshold;
public DifferenceDetector(int color, int threshold) {
super(color);
mThreshold = threshold * 3;
}
@Override
public boolean detectsColor(int R, int G, int B) {
return Math.abs(R - mR) + Math.abs(G - mG) + Math.abs(B - mB) <= mThreshold;
}
}
class RDistanceDetector extends AbstractColorDetector {
private final int mThreshold;
public RDistanceDetector(int color, int threshold) {
super(color);
mThreshold = threshold;
}
@Override
public boolean detectsColor(int R, int G, int B) {
return Math.abs(mR - R) <= mThreshold;
}
}
class RGBDistanceDetector extends AbstractColorDetector {
private final int mThreshold;
public RGBDistanceDetector(int color, int threshold) {
super(color);
mThreshold = threshold * threshold * 3;
}
@Override
public boolean detectsColor(int R, int G, int B) {
int dR = R - mR;
int dG = G - mG;
int dB = B - mB;
int d = dR * dR + dG * dG + dB * dB;
return d <= mThreshold;
}
}
class WeightedRGBDistanceDetector extends AbstractColorDetector {
private final int mThreshold;
private final int mR, mG, mB;
public WeightedRGBDistanceDetector(int color, int threshold) {
super(color);
mR = (color & 0xff0000) >> 16;
mG = (color & 0x00ff00) >> 8;
mB = color & 0xff;
mThreshold = threshold * threshold * 8;
}
@Override
public boolean detectsColor(int R, int G, int B) {
int dR = R - mR;
int dG = G - mG;
int dB = B - mB;
double meanR = (mR + R) / 2;
double weightR = 2 + meanR / 256;
double weightG = 4.0;
double weightB = 2 + (255 - meanR) / 256;
return weightR * dR * dR + weightG * dG * dG + weightB * dB * dB <= mThreshold;
}
}
class HDistanceDetector extends AbstractColorDetector {
private final int mH;
private final int mThreshold;
public HDistanceDetector(int color, int threshold) {
super(color);
mH = getH(mR, mG, mB);
mThreshold = threshold;
}
@Override
public boolean detectsColor(int R, int G, int B) {
return Math.abs(mH - getH(R, G, B)) <= mThreshold;
}
private static int getH(int R, int G, int B) {
int max, min, H;
if (R > G) {
min = Math.min(G, B);
max = Math.max(R, B);
} else {
min = Math.min(R, B);
max = Math.max(G, B);
}
if (R == max) {
H = (G - B) / (max - min) * 60;
} else if (G == max) {
H = 120 + (B - R) / (max - min) * 60;
} else {
H = 240 + (R - G) / (max - min) * 60;
}
if (H < 0) H = H + 360;
return H;
}
}
class HSDistanceDetector extends AbstractColorDetector {
private final int mH, mS;
private final int mThreshold;
public HSDistanceDetector(int color, int threshold) {
super(color);
long HS = getHS(mR, mG, mB);
mH = (int) (HS & 0xffffffffL);
mS = (int) ((HS >> 32) & 0xffffffffL);
mThreshold = threshold * 3729600 / 255;
}
public HSDistanceDetector(int color, float similarity) {
this(color, (int) (1.0f - similarity) * 255);
}
@Override
public boolean detectsColor(int R, int G, int B) {
long hs = getHS(R, G, B);
int dH = (int) (hs & 0xffffffffL) - mH;
int dS = (int) ((hs >> 32) & 0xffffffffL) - mS;
return dH * dH + dS * dS <= mThreshold;
}
private static long getHS(int R, int G, int B) {
int max, min, H;
if (R > G) {
min = Math.min(G, B);
max = Math.max(R, B);
} else {
min = Math.min(R, B);
max = Math.max(G, B);
}
if (R == max) {
H = (G - B) / (max - min) * 60;
} else if (G == max) {
H = 120 + (B - R) / (max - min) * 60;
} else {
H = 240 + (R - G) / (max - min) * 60;
}
if (H < 0) H = H + 360;
int S = (max - min) * 100 / max;
return H & ((long) S << 32);
}
}
}

267
autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java
View File

@@ -1,8 +1,6 @@
package com.stardust.autojs.core.image;
import android.graphics.Point;
import android.graphics.Rect;
import android.media.Image;
import android.graphics.Color;
import android.os.Build;
import android.support.annotation.RequiresApi;
@@ -10,6 +8,14 @@ import com.stardust.autojs.runtime.exception.ScriptInterruptedException;
import com.stardust.concurrent.VolatileBox;
import com.stardust.util.ScreenMetrics;
import org.opencv.android.Utils;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import java.util.List;
import java.util.Vector;
import java.util.concurrent.SynchronousQueue;
@@ -23,235 +29,50 @@ import java.util.concurrent.TimeUnit;
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
public class ColorFinder {
private static ThreadPoolExecutor sThreadPoolExecutor = new ThreadPoolExecutor(4, 16, 5, TimeUnit.MINUTES, new SynchronousQueue<Runnable>());
static {
sThreadPoolExecutor.allowCoreThreadTimeOut(true);
public static Point findColorEquals(ImageWrapper imageWrapper, int color) {
return findColorEquals(imageWrapper, color, null);
}
private ThreadPoolExecutor mThreadPoolExecutor;
private ScreenMetrics mScreenMetrics;
public ColorFinder(ThreadPoolExecutor threadPoolExecutor) {
mThreadPoolExecutor = threadPoolExecutor;
mScreenMetrics = new ScreenMetrics();
public static Point findColorEquals(ImageWrapper imageWrapper, int color, Rect region) {
return findColor(imageWrapper, color, 0, region);
}
public ColorFinder() {
this(sThreadPoolExecutor);
public static Point findColor(ImageWrapper imageWrapper, int color, int threshold) {
return findColor(imageWrapper, color, threshold, null);
}
public void prestartThreads() {
mThreadPoolExecutor.prestartAllCoreThreads();
}
public Point[] findAllColors(Image image, ColorDetector detector, Rect rect, int threadCount) {
List<Point> result = new Vector<>();
ColorIterator[] iterators = divide(image, rect, threadCount);
for (int i = 1; i < threadCount; i++) {
mThreadPoolExecutor.execute(new FindAllColorsRunnable(result, iterators[i], detector));
public static Point findColor(ImageWrapper imageWrapper, int color, int threshold, Rect region) {
Point[] points = findAllColors(imageWrapper, color, threshold, region);
if (points.length == 0) {
return null;
}
new FindAllColorsRunnable(result, iterators[0], detector).run();
Point[] points = new Point[result.size()];
for (int i = 0; i < points.length; i++) {
points[i] = scalePoint(result.get(i), image.getWidth(), image.getHeight());
return points[0];
}
public static Point[] findAllColors(ImageWrapper image, int color, int threshold, Rect rect) {
Mat bi = new Mat();
Scalar lowerBound = new Scalar(Color.red(color) - threshold, Color.green(color) - threshold,
Color.blue(color) - threshold, 255);
Scalar upperBound = new Scalar(Color.red(color) + threshold, Color.green(color) + threshold,
Color.blue(color) + threshold, 255);
if (rect != null) {
Core.inRange(new Mat(image.getMat(), rect), lowerBound, upperBound, bi);
} else {
Core.inRange(image.getMat(), lowerBound, upperBound, bi);
}
Mat nonZeroPos = new Mat();
Core.findNonZero(bi, nonZeroPos);
if (nonZeroPos.rows() == 0 || nonZeroPos.cols() == 0) {
return new Point[0];
}
Point[] points = new MatOfPoint(nonZeroPos).toArray();
if (rect != null) {
for (int i = 0; i < points.length; i++) {
points[i].x += rect.x;
points[i].y += rect.y;
}
}
return points;
}
public Point findColorConcurrently(Image image, ColorDetector detector, Rect rect, int threadCount) {
if (threadCount <= 1) {
return findColor(image, detector, rect);
}
VolatileBox<Point> result = new VolatileBox<>();
ColorIterator[] iterators = divide(image, rect, threadCount);
for (int i = 1; i < threadCount; i++) {
mThreadPoolExecutor.execute(new FindColorRunnable(result, iterators[i], detector));
}
new FindColorRunnable(result, iterators[0], detector).run();
return scalePoint(result.get(), image.getWidth(), image.getHeight());
}
private Point scalePoint(Point point, int width, int height) {
if (point == null)
return null;
mScreenMetrics.setDesignHeight(height);
mScreenMetrics.setDesignWidth(width);
point.set(mScreenMetrics.scaleX(point.x), mScreenMetrics.scaleY(point.y));
return point;
}
protected ColorIterator[] divide(Image image, Rect rect, int count) {
Rect[] subAreas = divideIntoSubAreas(rect, count);
int centerY = rect.centerY();
ColorIterator[] iterators = new ColorIterator[count];
for (int i = 1; i < subAreas.length; i++) {
Rect subArea = subAreas[i];
if (subArea.top > centerY) {
iterators[i] = new ColorIterator.SequentialIterator(image, subArea, true);
} else {
iterators[i] = new ColorIterator.SequentialIterator(image, subArea, true);
}
}
iterators[0] = new ColorIterator.SequentialIterator(image, subAreas[0], false);
return iterators;
}
protected Rect[] divideIntoSubAreas(Rect rect, int count) {
int row, column;
switch (count) {
case 4:
case 6:
case 8:
case 10:
case 14:
row = count / 2;
column = 2;
break;
case 9:
case 12:
case 15:
row = count / 3;
column = 3;
break;
case 16:
row = 4;
column = 4;
break;
default:
row = count;
column = 1;
}
Rect[] cells = new Rect[count];
int cellWidth = rect.width() / column;
int cellHeight = rect.height() / row;
for (int i = 0; i < row; i++) {
for (int j = 0; j < column; j++) {
int x = rect.left + j * cellWidth;
int y = rect.top + i * cellHeight;
cells[i * column + j] = new Rect(x, y, x + cellWidth, y + cellHeight);
}
}
return cells;
}
public Point findColorConcurrently(Image image, int color, Rect rect, int threadCount, int threshold) {
return findColorConcurrently(image, defaultColorDetector(color, threshold), rect, threadCount);
}
public Point findColorConcurrently(Image image, int color, Rect rect, int threadCount) {
return findColorConcurrently(image, color, rect, threadCount, 8);
}
public Point findColorConcurrently(Image image, int color, int threadCount) {
Rect rect = new Rect(0, 0, image.getWidth(), image.getHeight());
return findColorConcurrently(image, defaultColorDetector(color), rect, threadCount);
}
public Point findColorEqualsConcurrently(Image image, int color, Rect rect, int threadCount) {
return findColorConcurrently(image, new ColorDetector.EqualityDetector(color), rect, threadCount);
}
public static Point findColor(ColorIterator iterator, ColorDetector detector) {
Thread thread = Thread.currentThread();
ColorIterator.Pixel pixel = new ColorIterator.Pixel();
while (iterator.hasNext() && !thread.isInterrupted()) {
iterator.nextColor(pixel);
iterator.nextColor(pixel);
if (detector.detectsColor(pixel.red, pixel.green, pixel.blue)) {
return new Point(iterator.getX(), iterator.getY());
}
}
if (thread.isInterrupted()) {
throw new ScriptInterruptedException();
}
return null;
}
public ColorDetector defaultColorDetector(int color) {
return new ColorDetector.RGBDistanceDetector(color, 16);
}
public ColorDetector defaultColorDetector(int color, int threshold) {
return new ColorDetector.RGBDistanceDetector(color, threshold);
}
public ColorIterator defaultColorIterator(Image image, Rect rect) {
return new ColorIterator.SequentialIterator(image, rect);
}
public Point findColor(Image image, ColorDetector detector, Rect rect) {
return scalePoint(findColor(defaultColorIterator(image, rect), detector), image.getWidth(), image.getHeight());
}
public Point findColor(Image image, int color, Rect rect) {
return scalePoint(findColor(defaultColorIterator(image, rect), defaultColorDetector(color)), image.getWidth(), image.getHeight());
}
public Point findColor(Image image, int color) {
return findColor(image, color, new Rect(0, 0, image.getWidth(), image.getHeight()));
}
public Point findColorEquals(Image image, int color, Rect rect) {
return scalePoint(findColor(defaultColorIterator(image, rect), new ColorDetector.EqualityDetector(color)), image.getWidth(), image.getHeight());
}
private static class FindColorRunnable implements Runnable {
private final VolatileBox<Point> mResultBox;
private final ColorIterator mColorIterator;
private final ColorDetector mColorDetector;
private FindColorRunnable(VolatileBox<Point> resultBox, ColorIterator colorIterator, ColorDetector colorDetector) {
mResultBox = resultBox;
mColorIterator = colorIterator;
mColorDetector = colorDetector;
}
@Override
public void run() {
Thread thread = Thread.currentThread();
ColorIterator.Pixel pixel = new ColorIterator.Pixel();
while (mResultBox.isNull() && mColorIterator.hasNext() && !thread.isInterrupted()) {
mColorIterator.nextColor(pixel);
if (mColorDetector.detectsColor(pixel.red, pixel.green, pixel.blue)) {
mResultBox.set(new Point(mColorIterator.getX(), mColorIterator.getY()));
break;
}
}
if (thread.isInterrupted()) {
throw new ScriptInterruptedException();
}
}
}
private static class FindAllColorsRunnable implements Runnable {
private final List<Point> mResult;
private final ColorIterator mColorIterator;
private final ColorDetector mColorDetector;
private FindAllColorsRunnable(List<Point> result, ColorIterator colorIterator, ColorDetector colorDetector) {
mResult = result;
mColorIterator = colorIterator;
mColorDetector = colorDetector;
}
@Override
public void run() {
Thread thread = Thread.currentThread();
ColorIterator.Pixel pixel = new ColorIterator.Pixel();
while (mColorIterator.hasNext() && !thread.isInterrupted()) {
mColorIterator.nextColor(pixel);
if (mColorDetector.detectsColor(pixel.red, pixel.green, pixel.blue)) {
mResult.add(new Point(mColorIterator.getX(), mColorIterator.getY()));
}
}
if (thread.isInterrupted()) {
throw new ScriptInterruptedException();
}
}
}
}

187
autojs/src/main/java/com/stardust/autojs/core/image/ColorIterator.java
View File

@@ -1,187 +0,0 @@
package com.stardust.autojs.core.image;
import android.graphics.Rect;
import android.media.Image;
import java.nio.ByteBuffer;
/**
* Created by Stardust on 2017/5/20.
*/
public interface ColorIterator {
class Pixel {
int red;
int green;
int blue;
}
boolean hasNext();
void nextColor(Pixel pixel);
int getX();
int getY();
abstract class ImageColorIterator implements ColorIterator {
protected final ByteBuffer mByteBuffer;
protected final Rect mIterateArea;
public ImageColorIterator(Image image, Rect area, boolean duplicateBuffer) {
Image.Plane plane = image.getPlanes()[0];
if (duplicateBuffer) {
mByteBuffer = plane.getBuffer().duplicate();
} else {
mByteBuffer = plane.getBuffer();
}
mIterateArea = area;
}
public ImageColorIterator(Image image, Rect area) {
this(image, area, false);
}
protected void skip(int i) {
mByteBuffer.position(mByteBuffer.position() + i);
}
}
class SequentialIterator extends ImageColorIterator {
private static final String LOG_TAG = "SequentialIterator";
private final int mRowStride;
private final int mSkipPerRow;
private final int mWidth;
private final int mHeight;
private int mX = -1;
private int mY = 0;
public SequentialIterator(Image image, Rect area, boolean duplicateBuffer) {
super(image, area, duplicateBuffer);
Image.Plane plane = image.getPlanes()[0];
int pixelStride = plane.getPixelStride();
mRowStride = plane.getRowStride();
mWidth = area.width();
mHeight = area.height();
int rowPadding = mRowStride - pixelStride * image.getWidth();
mSkipPerRow = rowPadding + (image.getWidth() - mWidth) * pixelStride;
int offset = mIterateArea.top * mRowStride + mIterateArea.left * pixelStride;
mByteBuffer.position(offset);
}
public SequentialIterator(Image image, Rect area) {
this(image, area, false);
}
public SequentialIterator(Image image) {
this(image, new Rect(0, 0, image.getWidth(), image.getHeight()));
}
@Override
public boolean hasNext() {
return mX < mWidth - 1 || mY < mHeight - 1;
}
@Override
public int getX() {
return mIterateArea.left + mX;
}
@Override
public int getY() {
return mIterateArea.top + mY;
}
@Override
public void nextColor(Pixel pixel) {
if (mX == mWidth - 1) {
skip(mSkipPerRow);
mX = 0;
mY++;
} else {
mX++;
}
pixel.red = mByteBuffer.get() & 0xff;
pixel.green = mByteBuffer.get() & 0xff;
pixel.blue = mByteBuffer.get() & 0xff;
mByteBuffer.get();
}
}
// TODO: 2017/5/29 中心螺旋。未完成。
class CentralSpiralIterator extends ImageColorIterator {
private static final int DIRECTION_RIGHT = 0;
private static final int DIRECTION_TOP = 1;
private static final int DIRECTION_LEFT = 2;
private static final int DIRECTION_BOTTOM = 3;
private final int mPixelStride, mRowStride;
private int mNextStepSkip;
private int mStepCount;
private int mMaxStep = 1;
private int mDirection = DIRECTION_RIGHT;
public CentralSpiralIterator(Image image, Rect area, boolean duplicateBuffer) {
super(image, area, duplicateBuffer);
Image.Plane plane = image.getPlanes()[0];
mPixelStride = mNextStepSkip = plane.getPixelStride();
mRowStride = plane.getRowStride();
mByteBuffer.position(area.centerX() * mPixelStride + area.centerY() * mRowStride);
}
public CentralSpiralIterator(Image image, Rect area) {
this(image, area, false);
}
@Override
public boolean hasNext() {
return mByteBuffer.position() < mByteBuffer.limit();
}
@Override
public void nextColor(Pixel pixel) {
int c = mByteBuffer.getInt();
skip(mNextStepSkip);
mStepCount++;
if (mStepCount == mMaxStep) {
mStepCount = 0;
mMaxStep++;
mDirection = (mDirection + 1) & 4;
switch (mDirection) {
case DIRECTION_RIGHT:
mNextStepSkip = mPixelStride;
break;
case DIRECTION_TOP:
mNextStepSkip = -mRowStride;
break;
case DIRECTION_LEFT:
mNextStepSkip = -mPixelStride;
break;
case DIRECTION_BOTTOM:
mNextStepSkip = mRowStride;
break;
}
}
}
@Override
public int getX() {
return 0;
}
@Override
public int getY() {
return 0;
}
}
}

87
autojs/src/main/java/com/stardust/autojs/core/image/ConcurrentColorIterator.java
View File

@@ -1,87 +0,0 @@
package com.stardust.autojs.core.image;
import android.graphics.Rect;
import android.media.Image;
import java.nio.ByteBuffer;
/**
* Created by Stardust on 2017/5/21.
*/
public interface ConcurrentColorIterator {
void nextColor(Pixel pixel);
class Pixel {
public int x;
public int y;
public int color;
public boolean valid = true;
}
abstract class ConcurrentImageColorIterator implements ConcurrentColorIterator {
protected final ByteBuffer mByteBuffer;
protected final int mImageWidth, mImageHeight;
protected final Rect mIterateArea;
protected final int mAreaWidth, mAreaHeight;
protected volatile int mX = -1, mY;
public ConcurrentImageColorIterator(Image image, Rect area) {
Image.Plane plane = image.getPlanes()[0];
mByteBuffer = plane.getBuffer();
mImageWidth = image.getWidth();
mImageHeight = image.getHeight();
mIterateArea = area;
mAreaWidth = area.width();
mAreaHeight = area.height();
}
protected void skip(int i) {
mByteBuffer.position(mByteBuffer.position() + i);
}
}
class ConcurrentSequentialIterator extends ConcurrentImageColorIterator {
private final int mSkipPerRow;
public ConcurrentSequentialIterator(Image image, Rect area) {
super(image, area);
Image.Plane plane = image.getPlanes()[0];
int pixelStride = plane.getPixelStride();
int rowStride = plane.getRowStride();
int rowPadding = rowStride - pixelStride * mImageWidth;
mSkipPerRow = (mImageWidth - mAreaWidth) * pixelStride + rowPadding;
int offset = mIterateArea.top * rowStride + mIterateArea.left * pixelStride;
mByteBuffer.position(offset);
}
// TODO: 2017/5/21 对锁的竞争造成并发速度极慢。能否做到无锁？
@Override
public synchronized void nextColor(Pixel pixel) {
if (!(mY < mAreaHeight - 1 || mX < mAreaWidth - 1)) {
pixel.valid = false;
return;
}
int c = mByteBuffer.getInt();
c = ((c & 0xff) << 16) | (c & 0xff00) | ((c & 0xff0000) >> 16) | 0xff000000;
mX++;
if (mX == mAreaWidth) {
mX = 0;
mY++;
if (mSkipPerRow > 0) {
skip(mSkipPerRow);
}
}
pixel.x = mX + mIterateArea.left;
pixel.y = mY + mIterateArea.top;
pixel.color = c;
}
}
}

117
autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java Normal file
View File

@@ -0,0 +1,117 @@
package com.stardust.autojs.core.image;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.graphics.ImageFormat;
import android.media.Image;
import android.os.Build;
import android.support.annotation.RequiresApi;
import com.stardust.autojs.runtime.api.Images;
import com.stardust.pio.UncheckedIOException;
import org.opencv.android.Utils;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfInt;
import org.opencv.highgui.Highgui;
import org.opencv.imgproc.Imgproc;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.util.Collections;
/**
* Created by Stardust on 2017/11/25.
*/
public class ImageWrapper {
private Mat mMat;
private int mWidth;
private int mHeight;
private Bitmap mBitmap;
public ImageWrapper(Mat mat) {
mMat = mat;
mWidth = mat.cols();
mHeight = mat.rows();
}
public ImageWrapper(Bitmap bitmap) {
mBitmap = bitmap;
mWidth = bitmap.getWidth();
mHeight = bitmap.getHeight();
}
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
public static ImageWrapper ofImage(Image image) {
if (image == null) {
return null;
}
return new ImageWrapper(toBitmap(image));
}
public static ImageWrapper ofBitmap(Bitmap bitmap) {
if (bitmap == null) {
return null;
}
return new ImageWrapper(bitmap);
}
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
public static Bitmap toBitmap(Image image) {
Image.Plane plane = image.getPlanes()[0];
ByteBuffer buffer = plane.getBuffer();
buffer.position(0);
int pixelStride = plane.getPixelStride();
int rowPadding = plane.getRowStride() - pixelStride * image.getWidth();
Bitmap bitmap = Bitmap.createBitmap(image.getWidth() + rowPadding / pixelStride, image.getHeight(), Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
return bitmap;
}
public int getWidth() {
return mWidth;
}
public int getHeight() {
return mHeight;
}
public Mat getMat() {
if (mMat == null) {
mMat = new Mat();
Utils.bitmapToMat(mBitmap, mMat);
}
return mMat;
}
public void saveTo(String path) {
if (mBitmap != null) {
try {
mBitmap.compress(Bitmap.CompressFormat.PNG, 100, new FileOutputStream(path));
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
} else {
Highgui.imwrite(path, mMat);
}
}
public int getPixel(int x, int y) {
if (mBitmap != null) {
return mBitmap.getPixel(x, y);
}
double[] channels = mMat.get(x, y);
return Color.argb((int) channels[3], (int) channels[0], (int) channels[1], (int) channels[2]);
}
public Bitmap getBitmap() {
return mBitmap;
}
}

33
autojs/src/main/java/com/stardust/autojs/core/image/ScreenCapturer.java
View File

@@ -83,25 +83,26 @@ public class ScreenCapturer {
}
private void setImageListener(Handler handler) {
mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
if (mCachedImage != null) {
synchronized (mCachedImageLock) {
if (mCachedImage != null) {
mCachedImage.close();
}
mCachedImage = reader.acquireLatestImage();
return;
mImageReader.setOnImageAvailableListener(reader -> {
if (mCachedImage != null) {
synchronized (mCachedImageLock) {
if (mCachedImage != null) {
mCachedImage.close();
}
mCachedImage = reader.acquireLatestImage();
mCachedImageLock.notify();
return;
}
mCachedImage = reader.acquireLatestImage();
}
mCachedImage = reader.acquireLatestImage();
}, handler);
}
@Nullable
public Image capture() {
if (mUnderUsingImage == null && mCachedImage == null) {
waitForImageAvailable();
}
if (mCachedImage != null) {
if (mUnderUsingImage != null)
mUnderUsingImage.close();
@@ -113,6 +114,16 @@ public class ScreenCapturer {
return mUnderUsingImage;
}
private void waitForImageAvailable() {
synchronized (mCachedImageLock) {
try {
mCachedImageLock.wait();
} catch (InterruptedException e) {
throw new ScriptInterruptedException();
}
}
}
public int getScreenWidth() {
return mScreenWidth;
}

182
autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java Normal file
View File

@@ -0,0 +1,182 @@
package com.stardust.autojs.core.image;
import android.util.Pair;
import android.util.TimingLogger;
import com.stardust.util.Nath;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import java.util.ArrayList;
import java.util.List;
/**
* Created by Stardust on 2017/11/25.
*/
public class TemplateMatching {
private static final String LOG_TAG = "TemplateMatching";
public static final int MAX_LEVEL_AUTO = -1;
public static Point fastTemplateMatching(Mat img, Mat template, float threshold) {
return fastTemplateMatching(img, template, Imgproc.TM_CCOEFF_NORMED, 0.75f, threshold, MAX_LEVEL_AUTO);
}
public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel) {
TimingLogger logger = new TimingLogger(LOG_TAG, "fast_tm");
if (maxLevel == MAX_LEVEL_AUTO) {
maxLevel = selectPyramidLevel(img, template);
logger.addSplit("selectPyramidLevel:" + maxLevel);
}
Point p = null;
Mat matchResult;
double similarity = 0;
for (int level = maxLevel; level >= 0; level--) {
Mat src = getPyramidDownAtLevel(img, level);
Mat currentTemplate = getPyramidDownAtLevel(template, level);
if (p == null) {
if (!shouldContinueMatching(level, maxLevel)) {
break;
}
matchResult = matchTemplate(src, currentTemplate, matchMethod);
Pair<Point, Double> bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold);
p = bestMatched.first;
similarity = bestMatched.second;
} else {
Rect r = getROI(p, src, currentTemplate);
matchResult = matchTemplate(new Mat(src, r), currentTemplate, matchMethod);
Pair<Point, Double> bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold);
if (bestMatched.second < weakThreshold) {
p = null;
break;
}
p = bestMatched.first;
similarity = bestMatched.second;
p.x += r.x;
p.y += r.y;
if (bestMatched.second >= strictThreshold) {
pyrUp(p, level);
break;
}
}
logger.addSplit("level:" + level + " point:" + p);
}
logger.addSplit("result:" + p);
logger.dumpToLog();
if (similarity < strictThreshold) {
return null;
}
return p;
}
private static Mat getPyramidDownAtLevel(Mat m, int level) {
if (level == 0) {
return m;
}
int cols = m.cols();
int rows = m.rows();
for (int i = 0; i < level; i++) {
cols = (cols + 1) / 2;
rows = (rows + 1) / 2;
}
Mat r = new Mat(rows, cols, m.type());
Imgproc.resize(m, r, new Size(cols, rows));
return r;
}
private static void pyrUp(Point p, int level) {
for (int i = 0; i < level; i++) {
p.x *= 2;
p.y *= 2;
}
}
private static boolean shouldContinueMatching(int level, int maxLevel) {
if (level == maxLevel && level != 0) {
return true;
}
if (maxLevel <= 2) {
return false;
}
return level == maxLevel - 1;
}
private static Rect getROI(Point p, Mat src, Mat currentTemplate) {
int x = (int) (p.x * 2 - currentTemplate.rows() / 4);
x = Math.max(0, x);
int y = (int) (p.y * 2 - currentTemplate.cols() / 4);
y = Math.max(0, y);
int w = (int) (currentTemplate.rows() * 1.5);
int h = (int) (currentTemplate.cols() * 1.5);
if (x + w >= src.cols()) {
w = src.cols() - x - 1;
}
if (y + h >= src.rows()) {
h = src.rows() - y - 1;
}
return new Rect(x, y, w, h);
}
private static int selectPyramidLevel(Mat img, Mat template) {
int minDim = Nath.min(img.rows(), img.cols(), template.rows(), template.cols());
//这里选取12为图像缩小后的最小宽高，从而用log(2, minDim / 16)得到最多可以经过几次缩小。
int maxLevel = (int) (Math.log(minDim / 7) / Math.log(2));
if (maxLevel < 0) {
return 0;
}
//上限为6
return Math.min(6, maxLevel);
}
public static List<Mat> buildPyramid(Mat mat, int maxLevel) {
List<Mat> pyramid = new ArrayList<>();
pyramid.add(mat);
for (int i = 0; i < maxLevel; i++) {
Mat m = new Mat((mat.rows() + 1) / 2, (mat.cols() + 1) / 2, mat.type());
Imgproc.pyrDown(mat, m);
pyramid.add(m);
mat = m;
}
return pyramid;
}
public static Mat matchTemplate(Mat img, Mat temp, int match_method) {
int result_cols = img.cols() - temp.cols() + 1;
int result_rows = img.rows() - temp.rows() + 1;
Mat result = new Mat(result_rows, result_cols, CvType.CV_32FC1);
Imgproc.matchTemplate(img, temp, result, match_method);
return result;
}
public static Pair<Point, Double> getBestMatched(Mat tmResult, int matchMethod, float threshold) {
TimingLogger logger = new TimingLogger(LOG_TAG, "best_matched_point");
// FIXME: 2017/11/26 正交化?
// Core.normalize(tmResult, tmResult, 0, 1, Core.NORM_MINMAX, -1, new Mat());
Core.MinMaxLocResult mmr = Core.minMaxLoc(tmResult);
logger.addSplit("minMaxLoc");
double value;
Point pos;
if (matchMethod == Imgproc.TM_SQDIFF || matchMethod == Imgproc.TM_SQDIFF_NORMED) {
pos = mmr.minLoc;
value = -mmr.minVal;
} else {
pos = mmr.maxLoc;
value = mmr.maxVal;
}
logger.addSplit("value:" + value);
logger.dumpToLog();
return new Pair<>(pos, value);
}
}

4
autojs/src/main/java/com/stardust/autojs/core/inputevent/RootAutomator.java
View File

@@ -164,8 +164,8 @@ public class RootAutomator {
public void touchUp(int id) throws IOException {
sendEvent(EV_ABS, ABS_MT_TRACKING_ID, id);
sendEvent(EV_KEY, BTN_TOUCH, 0x00000000);
sendEvent(EV_KEY, BTN_TOOL_FINGER, 0x00000000);
// sendEvent(EV_KEY, BTN_TOUCH, 0x00000000);
// sendEvent(EV_KEY, BTN_TOOL_FINGER, 0x00000000);
sendEvent(EV_SYN, SYN_REPORT, 0x00000000);
}

23
autojs/src/main/java/com/stardust/autojs/execution/ScriptExecuteActivity.java
View File

@@ -60,16 +60,30 @@ public class ScriptExecuteActivity extends AppCompatActivity implements Thread.U
prepare();
doExecution();
} catch (Exception e) {
mExecutionListener.onException(mScriptExecution, e);
super.finish();
onException(e);
}
}
private void onException(Exception e) {
mExecutionListener.onException(mScriptExecution, e);
super.finish();
}
@SuppressWarnings("unchecked")
private void doExecution() {
mScriptEngine.setTag(ScriptEngine.TAG_SOURCE, mScriptSource);
mExecutionListener.onStart(mScriptExecution);
mResult = mScriptEngine.execute(mScriptSource);
((LoopBasedJavaScriptEngine) mScriptEngine).execute(mScriptSource, new LoopBasedJavaScriptEngine.ExecuteCallback() {
@Override
public void onResult(Object r) {
mResult = r;
}
@Override
public void onException(Exception e) {
ScriptExecuteActivity.this.onException(e);
}
});
}
private void prepare() {
@@ -94,8 +108,7 @@ public class ScriptExecuteActivity extends AppCompatActivity implements Thread.U
@Override
public void uncaughtException(Thread t, Throwable e) {
mExecutionListener.onException(mScriptExecution, (Exception) e);
super.finish();
onException((Exception) e);
}
private static class ActivityScriptExecution extends ScriptExecution.AbstractScriptExecution {

100
autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java
View File

@@ -6,6 +6,7 @@ import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Matrix;
import android.graphics.Point;
import android.media.Image;
import android.os.Build;
import android.os.Handler;
@@ -16,8 +17,10 @@ import android.view.Surface;
import android.view.WindowManager;
import com.stardust.autojs.core.image.ColorFinder;
import com.stardust.autojs.core.image.ImageWrapper;
import com.stardust.autojs.core.image.ScreenCaptureRequester;
import com.stardust.autojs.core.image.ScreenCapturer;
import com.stardust.autojs.core.image.TemplateMatching;
import com.stardust.autojs.runtime.ScriptRuntime;
import com.stardust.autojs.runtime.exception.ScriptInterruptedException;
import com.stardust.autojs.annotation.ScriptVariable;
@@ -25,6 +28,15 @@ import com.stardust.concurrent.VolatileBox;
import com.stardust.pio.UncheckedIOException;
import com.stardust.util.ScreenMetrics;
import org.opencv.android.Utils;
import org.opencv.contrib.FaceRecognizer;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.highgui.Highgui;
import org.opencv.imgproc.Imgproc;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
@@ -56,7 +68,6 @@ public class Images {
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public boolean requestScreenCapture(final int width, final int height) {
mScriptRuntime.requiresApi(21);
colorFinder.prestartThreads();
final VolatileBox<Boolean> requestResult = new VolatileBox<>();
mScreenCaptureRequester.setOnActivityResultCallback(new ScreenCaptureRequester.Callback() {
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
@@ -84,19 +95,17 @@ public class Images {
}
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public Image captureScreen() {
public ImageWrapper captureScreen() {
mScriptRuntime.requiresApi(21);
if (mScreenCapturer == null) {
throw new SecurityException("No screen capture permission");
}
colorFinder.prestartThreads();
return mScreenCapturer.capture();
return ImageWrapper.ofImage(mScreenCapturer.capture());
}
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public boolean captureScreen(String path) {
mScriptRuntime.requiresApi(21);
Image image = mScreenCapturer.capture();
ImageWrapper image = captureScreen();
if (image != null) {
saveImage(image, path);
return true;
@@ -104,22 +113,10 @@ public class Images {
return false;
}
public void saveImage(Image image, String path) {
Bitmap bitmap = toBitmap(image);
saveBitmap(bitmap, path);
bitmap.recycle();
public void saveImage(ImageWrapper image, String path) {
image.saveTo(path);
}
public static Bitmap toBitmap(Image image) {
Image.Plane plane = image.getPlanes()[0];
ByteBuffer buffer = plane.getBuffer();
buffer.position(0);
int pixelStride = plane.getPixelStride();
int rowPadding = plane.getRowStride() - pixelStride * image.getWidth();
Bitmap bitmap = Bitmap.createBitmap(image.getWidth() + rowPadding / pixelStride, image.getHeight(), Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
return bitmap;
}
public static int pixel(Image image, int x, int y) {
int originX = x;
@@ -134,15 +131,15 @@ public class Images {
return (c & 0xff000000) + ((c & 0xff) << 16) + (c & 0x00ff00) + ((c & 0xff0000) >> 16);
}
public static int pixel(Bitmap bitmap, int x, int y) {
x = ScreenMetrics.rescaleX(x, bitmap.getWidth());
y = ScreenMetrics.rescaleY(y, bitmap.getHeight());
return bitmap.getPixel(x, y);
public static int pixel(ImageWrapper image, int x, int y) {
x = ScreenMetrics.rescaleX(x, image.getWidth());
y = ScreenMetrics.rescaleY(y, image.getHeight());
return image.getPixel(x, y);
}
public static Bitmap read(String path) {
return BitmapFactory.decodeFile(path);
public ImageWrapper read(String path) {
return ImageWrapper.ofBitmap(BitmapFactory.decodeFile(path));
}
public static void saveBitmap(Bitmap bitmap, String path) {
@@ -153,24 +150,6 @@ public class Images {
}
}
public static void saveBitmap(Bitmap bitmap, String path, int width, int height) {
if (width != bitmap.getWidth() || height != bitmap.getHeight()) {
Bitmap scaleBitmap = scaleBitmap(bitmap, width, height);
saveBitmap(scaleBitmap, path);
if (scaleBitmap != bitmap) {
scaleBitmap.recycle();
}
} else {
saveBitmap(bitmap, path);
}
}
public void saveImage(Image image, String path, int width, int height) {
Bitmap bitmap = toBitmap(image);
saveBitmap(bitmap, path, width, height);
bitmap.recycle();
}
public static Bitmap scaleBitmap(Bitmap origin, int newWidth, int newHeight) {
if (origin == null) {
return null;
@@ -190,4 +169,37 @@ public class Images {
}
}
public Point findImage(ImageWrapper image, ImageWrapper template) {
return findImage(image, template, 0.9f, null);
}
public Point findImage(ImageWrapper image, ImageWrapper template, float threshold) {
return findImage(image, template, threshold, null);
}
public Point findImage(ImageWrapper image, ImageWrapper template, float threshold, Rect rect) {
return findImage(image, template, threshold, rect, TemplateMatching.MAX_LEVEL_AUTO);
}
public Point findImage(ImageWrapper image, ImageWrapper template, float threshold, Rect rect, int maxLevel) {
Mat src = image.getMat();
if (rect != null) {
src = new Mat(src, rect);
}
org.opencv.core.Point point = TemplateMatching.fastTemplateMatching(src, template.getMat(), threshold);
if (point != null && rect != null) {
point.x += rect.x;
point.y += rect.y;
}
return toAndroidPoint(point);
}
public static Point toAndroidPoint(org.opencv.core.Point p) {
if (p == null) {
return null;
}
return new Point((int) p.x, (int) p.y);
}
}

11
autojs/src/main/java/com/stardust/autojs/runtime/exception/ScriptInterruptedException.java
View File

@@ -1,12 +1,13 @@
package com.stardust.autojs.runtime.exception;
/**
* Created by Stardust on 2017/4/30.
*/
public class ScriptInterruptedException extends ScriptException {
public ScriptInterruptedException(){
public ScriptInterruptedException() {
}
@@ -15,7 +16,13 @@ public class ScriptInterruptedException extends ScriptException {
}
public static boolean causedByInterrupted(Throwable e) {
return e instanceof ScriptInterruptedException || e.getCause() instanceof ScriptInterruptedException;
while (e != null) {
if (e instanceof ScriptInterruptedException || e instanceof InterruptedException) {
return true;
}
e = e.getCause();
}
return false;
}
}