diff --git a/.idea/modules.xml b/.idea/modules.xml index 7e950955..58fc8a76 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -2,9 +2,7 @@ - - - + diff --git a/app/build.gradle b/app/build.gradle index 86960cc8..ebbc5118 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -9,8 +9,8 @@ android { applicationId "com.stardust.scriptdroid" minSdkVersion 17 targetSdkVersion 23 - versionCode 221 - versionName "3.0.0 Alpha21" + versionCode 222 + versionName "3.0.0 Alpha22" testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" multiDexEnabled true ndk { diff --git a/app/src/main/assets/sample/图片与图色处理/区域找色1.js b/app/src/main/assets/sample/图片与图色处理/区域找色1.js index 06fb31dd..067834dc 100644 --- a/app/src/main/assets/sample/图片与图色处理/区域找色1.js +++ b/app/src/main/assets/sample/图片与图色处理/区域找色1.js @@ -3,11 +3,10 @@ if(!requestScreenCapture()){ stop(); } var img = captureScreen(); -//0xffffff为白色 toastLog("开始找色"); -//指定在位置(90, 220)宽高为900*1000的区域找色。 -//0xff00cc是编辑器的深粉红色字体(字符串)颜色 -var point = findColorInRegion(img, "#ff00cc", 90, 220, 900, 1000); +//指定在位置(100, 220)宽高为400*400的区域找色。 +//#75438a是编辑器默认主题的棕红色字体(数字)颜色，位置大约在第5行的"2000"，坐标大约为(283, 465) +var point = findColorInRegion(img, "#75438a", 90, 220, 900, 1000); if(point){ toastLog("x = " + point.x + ", y = " + point.y); }else{ diff --git a/app/src/main/assets/sample/图片与图色处理/精确找色.js b/app/src/main/assets/sample/图片与图色处理/精确找色.js index b37e7eb5..5c1c81d3 100644 --- a/app/src/main/assets/sample/图片与图色处理/精确找色.js +++ b/app/src/main/assets/sample/图片与图色处理/精确找色.js @@ -4,8 +4,8 @@ if(!requestScreenCapture()){ } var img = captureScreen(); toastLog("开始找色"); -//0x006699为编辑器蓝色字体(var)的颜色 -//找到颜色与0x006699完全相等的颜色 +//0x1d75b3为编辑器默认主题蓝色字体(if, var等关键字)的颜色 +//找到颜色与0x1d75b3完全相等的颜色 var point = findColorEquals(img, 0x006699); if(point){ toastLog("x = " + point.x + ", y = " + point.y); diff --git a/app/src/main/java/com/stardust/scriptdroid/autojs/AutoJs.java b/app/src/main/java/com/stardust/scriptdroid/autojs/AutoJs.java index ed871681..7e153e1e 100644 --- a/app/src/main/java/com/stardust/scriptdroid/autojs/AutoJs.java +++ b/app/src/main/java/com/stardust/scriptdroid/autojs/AutoJs.java @@ -44,6 +44,9 @@ import com.stardust.scriptdroid.tool.AccessibilityServiceTool; import com.stardust.view.accessibility.LayoutInspector; import com.stardust.view.accessibility.NotificationListener; +import org.opencv.android.BaseLoaderCallback; +import org.opencv.android.OpenCVLoader; + /** * Created by Stardust on 2017/4/2. @@ -78,7 +81,7 @@ public class AutoJs { mContext = context; mUiHandler = new UiHandler(context); mAppUtils = new AppUtils(context); - mGlobalConsole = new GlobalStardustConsole(mUiHandler){ + mGlobalConsole = new GlobalStardustConsole(mUiHandler) { @Override public String println(int level, CharSequence charSequence) { String log = super.println(level, charSequence); @@ -93,6 +96,8 @@ public class AutoJs { mScriptEngineService.registerGlobalScriptExecutionListener(new ScriptExecutionGlobalListener()); registerActivityLifecycleCallbacks(); InputEventObserver.initGlobal(context); + OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_13, context, new BaseLoaderCallback(context) { + }); } private ScriptEngineService buildScriptEngineService() { diff --git a/app/src/main/jniLibs/armeabi-v7a/libopencv_core.a b/app/src/main/jniLibs/armeabi-v7a/libopencv_core.a new file mode 100644 index 00000000..2e4032e9 Binary files /dev/null and b/app/src/main/jniLibs/armeabi-v7a/libopencv_core.a differ diff --git a/app/src/main/jniLibs/armeabi-v7a/libopencv_imgproc.a b/app/src/main/jniLibs/armeabi-v7a/libopencv_imgproc.a new file mode 100644 index 00000000..f4f66b70 Binary files /dev/null and b/app/src/main/jniLibs/armeabi-v7a/libopencv_imgproc.a differ diff --git a/autojs/build.gradle b/autojs/build.gradle index 3c3f0825..0289e85a 100644 --- a/autojs/build.gradle +++ b/autojs/build.gradle @@ -51,6 +51,7 @@ dependencies { compile 'com.android.support:design:25.4.0' compile 'com.android.support:appcompat-v7:25.4.0' compile 'com.github.hyb1996:EnhancedFloaty:0.17' + compile 'com.github.hyb1996:OpenCvLib:2.4.13.4-imgproc' // Terminal emulator compile(name: 'libtermexec-release', ext: 'aar') compile(name: 'emulatorview-release', ext: 'aar') diff --git a/autojs/src/main/assets/modules/__http__.js b/autojs/src/main/assets/modules/__http__.js index 1dde8abf..a965b035 100644 --- a/autojs/src/main/assets/modules/__http__.js +++ b/autojs/src/main/assets/modules/__http__.js @@ -37,7 +37,7 @@ module.exports = function(runtime, scope){ } call.enqueue(new Callback({ onResponse: function(call, res){ - callback(res); + callback(wrapResponse(res)); }, onFailure: function(call, ex){ callback(null, ex); @@ -103,6 +103,7 @@ module.exports = function(runtime, scope){ function wrapResponse(res){ var r = {}; r.statusCode = res.code(); + r.statusMessage = res.message(); var headers = res.headers(); r.headers = {}; for(var i = 0; i < headers.size(); i++){ @@ -112,6 +113,7 @@ module.exports = function(runtime, scope){ r.body.json = function(){ return JSON.parse(r.body.string()); } + r.body.contentType = r.body.contentType(); r.request = res.request(); r.url = r.request.url(); r.method = r.request.method(); diff --git a/autojs/src/main/assets/modules/__images__.js b/autojs/src/main/assets/modules/__images__.js index 68597f3f..196a0ab2 100644 --- a/autojs/src/main/assets/modules/__images__.js +++ b/autojs/src/main/assets/modules/__images__.js @@ -17,13 +17,15 @@ module.exports = function(__runtime__, scope){ images.captureScreen = rtImages.captureScreen.bind(rtImages); + images.read = rtImages.read.bind(rtImages); + images.saveImage = rtImages.saveImage.bind(rtImages); images.pixel = rtImages.pixel; images.detectsColor = function(img, color, x, y, threshold, algorithm){ color = parseColor(color); - algorithm = algorithm || "rgb"; + algorithm = algorithm || "diff"; threshold = threshold || 16; var colorDetector = getColorDetector(color, algorithm, threshold); var pixel = images.pixel(img, x, y); @@ -34,54 +36,80 @@ module.exports = function(__runtime__, scope){ color = parseColor(color); options = options || {}; var region = options.region || []; - var x = region[0] || 0; - var y = region[1] || 0; - var width = region[2] || (img.getWidth() - x); - var height = region[3] || (img.getHeight() - y); - var threads = options.threads || 2; if(options.similarity){ var threshold = parseInt(255 * (1 - options.similarity)); }else{ var threshold = options.threshold || 16; } - algorithm = options.algorithm || "rgb"; - var rect = new android.graphics.Rect(x, y, width + x, height + y); - var colorDetector = getColorDetector(color, algorithm, threshold); - return colorFinder.findColorConcurrently(img, colorDetector, rect, threads); + if(options.region){ + return rtImages.toAndroidPoint(colorFinder.findColor(img, color, threshold, buildRegion(options.region, img))); + }else{ + return rtImages.toAndroidPoint(colorFinder.findColor(img, color, threshold, null)); + } } - images.findColorInRegion = function(img, color, x, y, width, height, threads, algorithm, threshold){ + images.findColorInRegion = function(img, color, x, y, width, height, threshold){ return findColor(img, color, { region: [x, y, width, height], - algorithm: algorithm, - threshold: threshold, - threads: threads + threshold: threshold }); } - images.findColorEquals = function(img, color, x, y, width, height, threads){ + images.findColorEquals = function(img, color, x, y, width, height){ return findColor(img, color, { region: [x, y, width, height], - algorithm: "equal", - threads: threads + threshold: 0 }); } - function getColorDetector(color, algorithm, threshold){ - switch(algorithm){ - case "rgb": - return new com.stardust.autojs.core.image.ColorDetector.RGBDistanceDetector(color, threshold); - case "equal": - return new com.stardust.autojs.core.image.ColorDetector.EqualityDetector(color); - case "diff": - return new com.stardust.autojs.core.image.ColorDetector.DifferenceDetector(color, threshold); - case "rgb+": - return new com.stardust.autojs.core.image.ColorDetector.WeightedRGBDistanceDetector(color, threshold); - case "hs": - return new com.stardust.autojs.core.image.ColorDetector.HSDistanceDetector(color, threshold); - } - throw new Error("Unknown algorithm: " + algorithm); - } + images.findColors = function(img, color, options){ + color = parseColor(color); + options = options || {}; + if(options.similarity){ + var threshold = parseInt(255 * (1 - options.similarity)); + }else{ + var threshold = options.threshold || 16; + } + if(options.region){ + return toPointArray(colorFinder.findAllColors(img, color, threshold, buildRegion(options.region, img))); + }else{ + return toPointArray(colorFinder.findAllColors(img, color, threshold, null)); + } + } + + images.findImage = function(img, template, options){ + options = options || {}; + var threshold = options.threshold || 0.9; + var maxLevel = options.level || -1; + if(options.region){ + return rtImages.findImage(img, template, threshold, buildRegion(options, img), maxLevel); + }else{ + return rtImages.findImage(img, template, threshold, null, maxLevel); + } + } + + images.findImageInRegion = function(img, template, x, y, width, height, threshold){ + return images.findImage(img, template, { + region: [x, y, width, height], + threshold: threshold + }); + } + + function toPointArray(points){ + var arr = []; + for(var i = 0; i < points.length; i++){ + arr.push(rtImages.toAndroidPoint(points[i])); + } + return arr; + } + + function buildRegion(region, img){ + var x = region[0] || 0; + var y = region[1] || 0; + var width = region[2] || (img.getWidth() - x); + var height = region[3] || (img.getHeight() - y); + return new org.opencv.core.Rect(x, y, width, height); + } function parseColor(color){ if(typeof(color) == 'string'){ @@ -94,7 +122,7 @@ module.exports = function(__runtime__, scope){ return color; } - scope.__asGlobal__(images, ['requestScreenCapture', 'captureScreen', 'findColor', 'findColorInRegion', 'findColorEquals']); + scope.__asGlobal__(images, ['requestScreenCapture', 'captureScreen', 'findImage', 'findImageInRegion', 'findColor', 'findColorInRegion', 'findColorEquals']); scope.colors = colors; diff --git a/autojs/src/main/assets/modules/__util__.js b/autojs/src/main/assets/modules/__util__.js index 4bca9677..49e37c47 100644 --- a/autojs/src/main/assets/modules/__util__.js +++ b/autojs/src/main/assets/modules/__util__.js @@ -24,8 +24,8 @@ exports.format = function(f) { if (!isString(f)) { var objects = []; for (var i = 0; i < arguments.length; i++) { - var v = isJavaObject(arguments[i]) ? arguments[i].toString() : - arguments[i]; + var v = (arguments[i] && isJavaObject(arguments[i])) ? arguments[i].toString() : + arguments[i]; objects.push(inspect(v)); } return objects.join(' '); diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ColorDetector.java b/autojs/src/main/java/com/stardust/autojs/core/image/ColorDetector.java deleted file mode 100644 index 76a1fef4..00000000 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ColorDetector.java +++ /dev/null @@ -1,199 +0,0 @@ -package com.stardust.autojs.core.image; - -import android.graphics.Color; - -/** - * Created by Stardust on 2017/5/20. - */ - -public interface ColorDetector { - - boolean detectsColor(int red, int green, int blue); - - abstract class AbstractColorDetector implements ColorDetector { - - protected final int mColor; - protected final int mR, mG, mB; - - public AbstractColorDetector(int color) { - mColor = color; - mR = Color.red(color); - mG = Color.green(color); - mB = Color.blue(color); - } - } - - class EqualityDetector extends AbstractColorDetector { - - - public EqualityDetector(int color) { - super(color); - } - - @Override - public boolean detectsColor(int red, int green, int blue) { - return mR == red && mG == green && mB == blue; - } - } - - class DifferenceDetector extends AbstractColorDetector { - - private final int mThreshold; - - public DifferenceDetector(int color, int threshold) { - super(color); - mThreshold = threshold * 3; - } - - - @Override - public boolean detectsColor(int R, int G, int B) { - return Math.abs(R - mR) + Math.abs(G - mG) + Math.abs(B - mB) <= mThreshold; - } - } - - class RDistanceDetector extends AbstractColorDetector { - - private final int mThreshold; - - public RDistanceDetector(int color, int threshold) { - super(color); - mThreshold = threshold; - } - - @Override - public boolean detectsColor(int R, int G, int B) { - return Math.abs(mR - R) <= mThreshold; - } - } - - class RGBDistanceDetector extends AbstractColorDetector { - - private final int mThreshold; - - public RGBDistanceDetector(int color, int threshold) { - super(color); - mThreshold = threshold * threshold * 3; - } - - @Override - public boolean detectsColor(int R, int G, int B) { - int dR = R - mR; - int dG = G - mG; - int dB = B - mB; - int d = dR * dR + dG * dG + dB * dB; - return d <= mThreshold; - } - } - - class WeightedRGBDistanceDetector extends AbstractColorDetector { - - private final int mThreshold; - private final int mR, mG, mB; - - public WeightedRGBDistanceDetector(int color, int threshold) { - super(color); - mR = (color & 0xff0000) >> 16; - mG = (color & 0x00ff00) >> 8; - mB = color & 0xff; - mThreshold = threshold * threshold * 8; - } - - @Override - public boolean detectsColor(int R, int G, int B) { - int dR = R - mR; - int dG = G - mG; - int dB = B - mB; - double meanR = (mR + R) / 2; - double weightR = 2 + meanR / 256; - double weightG = 4.0; - double weightB = 2 + (255 - meanR) / 256; - return weightR * dR * dR + weightG * dG * dG + weightB * dB * dB <= mThreshold; - } - } - - class HDistanceDetector extends AbstractColorDetector { - - private final int mH; - private final int mThreshold; - - public HDistanceDetector(int color, int threshold) { - super(color); - mH = getH(mR, mG, mB); - mThreshold = threshold; - } - - @Override - public boolean detectsColor(int R, int G, int B) { - return Math.abs(mH - getH(R, G, B)) <= mThreshold; - } - - private static int getH(int R, int G, int B) { - int max, min, H; - if (R > G) { - min = Math.min(G, B); - max = Math.max(R, B); - } else { - min = Math.min(R, B); - max = Math.max(G, B); - } - if (R == max) { - H = (G - B) / (max - min) * 60; - } else if (G == max) { - H = 120 + (B - R) / (max - min) * 60; - } else { - H = 240 + (R - G) / (max - min) * 60; - } - if (H < 0) H = H + 360; - return H; - } - } - - class HSDistanceDetector extends AbstractColorDetector { - - private final int mH, mS; - private final int mThreshold; - - public HSDistanceDetector(int color, int threshold) { - super(color); - long HS = getHS(mR, mG, mB); - mH = (int) (HS & 0xffffffffL); - mS = (int) ((HS >> 32) & 0xffffffffL); - mThreshold = threshold * 3729600 / 255; - } - - public HSDistanceDetector(int color, float similarity) { - this(color, (int) (1.0f - similarity) * 255); - } - - @Override - public boolean detectsColor(int R, int G, int B) { - long hs = getHS(R, G, B); - int dH = (int) (hs & 0xffffffffL) - mH; - int dS = (int) ((hs >> 32) & 0xffffffffL) - mS; - return dH * dH + dS * dS <= mThreshold; - } - - private static long getHS(int R, int G, int B) { - int max, min, H; - if (R > G) { - min = Math.min(G, B); - max = Math.max(R, B); - } else { - min = Math.min(R, B); - max = Math.max(G, B); - } - if (R == max) { - H = (G - B) / (max - min) * 60; - } else if (G == max) { - H = 120 + (B - R) / (max - min) * 60; - } else { - H = 240 + (R - G) / (max - min) * 60; - } - if (H < 0) H = H + 360; - int S = (max - min) * 100 / max; - return H & ((long) S << 32); - } - } - -} \ No newline at end of file diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java b/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java index ac9053ad..f68d7cad 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java @@ -1,8 +1,6 @@ package com.stardust.autojs.core.image; -import android.graphics.Point; -import android.graphics.Rect; -import android.media.Image; +import android.graphics.Color; import android.os.Build; import android.support.annotation.RequiresApi; @@ -10,6 +8,14 @@ import com.stardust.autojs.runtime.exception.ScriptInterruptedException; import com.stardust.concurrent.VolatileBox; import com.stardust.util.ScreenMetrics; +import org.opencv.android.Utils; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.core.Point; +import org.opencv.core.Rect; +import org.opencv.core.Scalar; + import java.util.List; import java.util.Vector; import java.util.concurrent.SynchronousQueue; @@ -23,235 +29,50 @@ import java.util.concurrent.TimeUnit; @RequiresApi(api = Build.VERSION_CODES.KITKAT) public class ColorFinder { - private static ThreadPoolExecutor sThreadPoolExecutor = new ThreadPoolExecutor(4, 16, 5, TimeUnit.MINUTES, new SynchronousQueue()); - - static { - sThreadPoolExecutor.allowCoreThreadTimeOut(true); + public static Point findColorEquals(ImageWrapper imageWrapper, int color) { + return findColorEquals(imageWrapper, color, null); } - private ThreadPoolExecutor mThreadPoolExecutor; - private ScreenMetrics mScreenMetrics; - - public ColorFinder(ThreadPoolExecutor threadPoolExecutor) { - mThreadPoolExecutor = threadPoolExecutor; - mScreenMetrics = new ScreenMetrics(); + public static Point findColorEquals(ImageWrapper imageWrapper, int color, Rect region) { + return findColor(imageWrapper, color, 0, region); } - public ColorFinder() { - this(sThreadPoolExecutor); + public static Point findColor(ImageWrapper imageWrapper, int color, int threshold) { + return findColor(imageWrapper, color, threshold, null); } - public void prestartThreads() { - mThreadPoolExecutor.prestartAllCoreThreads(); - } - - public Point[] findAllColors(Image image, ColorDetector detector, Rect rect, int threadCount) { - List result = new Vector<>(); - ColorIterator[] iterators = divide(image, rect, threadCount); - for (int i = 1; i < threadCount; i++) { - mThreadPoolExecutor.execute(new FindAllColorsRunnable(result, iterators[i], detector)); + public static Point findColor(ImageWrapper imageWrapper, int color, int threshold, Rect region) { + Point[] points = findAllColors(imageWrapper, color, threshold, region); + if (points.length == 0) { + return null; } - new FindAllColorsRunnable(result, iterators[0], detector).run(); - Point[] points = new Point[result.size()]; - for (int i = 0; i < points.length; i++) { - points[i] = scalePoint(result.get(i), image.getWidth(), image.getHeight()); + return points[0]; + } + + public static Point[] findAllColors(ImageWrapper image, int color, int threshold, Rect rect) { + Mat bi = new Mat(); + Scalar lowerBound = new Scalar(Color.red(color) - threshold, Color.green(color) - threshold, + Color.blue(color) - threshold, 255); + Scalar upperBound = new Scalar(Color.red(color) + threshold, Color.green(color) + threshold, + Color.blue(color) + threshold, 255); + if (rect != null) { + Core.inRange(new Mat(image.getMat(), rect), lowerBound, upperBound, bi); + } else { + Core.inRange(image.getMat(), lowerBound, upperBound, bi); + } + Mat nonZeroPos = new Mat(); + Core.findNonZero(bi, nonZeroPos); + if (nonZeroPos.rows() == 0 || nonZeroPos.cols() == 0) { + return new Point[0]; + } + Point[] points = new MatOfPoint(nonZeroPos).toArray(); + if (rect != null) { + for (int i = 0; i < points.length; i++) { + points[i].x += rect.x; + points[i].y += rect.y; + } } return points; } - public Point findColorConcurrently(Image image, ColorDetector detector, Rect rect, int threadCount) { - if (threadCount <= 1) { - return findColor(image, detector, rect); - } - VolatileBox result = new VolatileBox<>(); - ColorIterator[] iterators = divide(image, rect, threadCount); - for (int i = 1; i < threadCount; i++) { - mThreadPoolExecutor.execute(new FindColorRunnable(result, iterators[i], detector)); - } - new FindColorRunnable(result, iterators[0], detector).run(); - return scalePoint(result.get(), image.getWidth(), image.getHeight()); - } - - - private Point scalePoint(Point point, int width, int height) { - if (point == null) - return null; - mScreenMetrics.setDesignHeight(height); - mScreenMetrics.setDesignWidth(width); - point.set(mScreenMetrics.scaleX(point.x), mScreenMetrics.scaleY(point.y)); - return point; - } - - protected ColorIterator[] divide(Image image, Rect rect, int count) { - Rect[] subAreas = divideIntoSubAreas(rect, count); - int centerY = rect.centerY(); - ColorIterator[] iterators = new ColorIterator[count]; - for (int i = 1; i < subAreas.length; i++) { - Rect subArea = subAreas[i]; - if (subArea.top > centerY) { - iterators[i] = new ColorIterator.SequentialIterator(image, subArea, true); - } else { - iterators[i] = new ColorIterator.SequentialIterator(image, subArea, true); - } - } - iterators[0] = new ColorIterator.SequentialIterator(image, subAreas[0], false); - return iterators; - } - - protected Rect[] divideIntoSubAreas(Rect rect, int count) { - int row, column; - switch (count) { - case 4: - case 6: - case 8: - case 10: - case 14: - row = count / 2; - column = 2; - break; - case 9: - case 12: - case 15: - row = count / 3; - column = 3; - break; - case 16: - row = 4; - column = 4; - break; - default: - row = count; - column = 1; - } - Rect[] cells = new Rect[count]; - int cellWidth = rect.width() / column; - int cellHeight = rect.height() / row; - for (int i = 0; i < row; i++) { - for (int j = 0; j < column; j++) { - int x = rect.left + j * cellWidth; - int y = rect.top + i * cellHeight; - cells[i * column + j] = new Rect(x, y, x + cellWidth, y + cellHeight); - } - } - return cells; - } - - public Point findColorConcurrently(Image image, int color, Rect rect, int threadCount, int threshold) { - return findColorConcurrently(image, defaultColorDetector(color, threshold), rect, threadCount); - } - - public Point findColorConcurrently(Image image, int color, Rect rect, int threadCount) { - return findColorConcurrently(image, color, rect, threadCount, 8); - } - - public Point findColorConcurrently(Image image, int color, int threadCount) { - Rect rect = new Rect(0, 0, image.getWidth(), image.getHeight()); - return findColorConcurrently(image, defaultColorDetector(color), rect, threadCount); - } - - public Point findColorEqualsConcurrently(Image image, int color, Rect rect, int threadCount) { - return findColorConcurrently(image, new ColorDetector.EqualityDetector(color), rect, threadCount); - } - - public static Point findColor(ColorIterator iterator, ColorDetector detector) { - Thread thread = Thread.currentThread(); - ColorIterator.Pixel pixel = new ColorIterator.Pixel(); - while (iterator.hasNext() && !thread.isInterrupted()) { - iterator.nextColor(pixel); - iterator.nextColor(pixel); - if (detector.detectsColor(pixel.red, pixel.green, pixel.blue)) { - return new Point(iterator.getX(), iterator.getY()); - } - } - if (thread.isInterrupted()) { - throw new ScriptInterruptedException(); - } - return null; - } - - public ColorDetector defaultColorDetector(int color) { - return new ColorDetector.RGBDistanceDetector(color, 16); - } - - public ColorDetector defaultColorDetector(int color, int threshold) { - return new ColorDetector.RGBDistanceDetector(color, threshold); - } - - public ColorIterator defaultColorIterator(Image image, Rect rect) { - return new ColorIterator.SequentialIterator(image, rect); - } - - public Point findColor(Image image, ColorDetector detector, Rect rect) { - return scalePoint(findColor(defaultColorIterator(image, rect), detector), image.getWidth(), image.getHeight()); - } - - public Point findColor(Image image, int color, Rect rect) { - return scalePoint(findColor(defaultColorIterator(image, rect), defaultColorDetector(color)), image.getWidth(), image.getHeight()); - } - - public Point findColor(Image image, int color) { - return findColor(image, color, new Rect(0, 0, image.getWidth(), image.getHeight())); - } - - public Point findColorEquals(Image image, int color, Rect rect) { - return scalePoint(findColor(defaultColorIterator(image, rect), new ColorDetector.EqualityDetector(color)), image.getWidth(), image.getHeight()); - } - - private static class FindColorRunnable implements Runnable { - - private final VolatileBox mResultBox; - private final ColorIterator mColorIterator; - private final ColorDetector mColorDetector; - - private FindColorRunnable(VolatileBox resultBox, ColorIterator colorIterator, ColorDetector colorDetector) { - mResultBox = resultBox; - mColorIterator = colorIterator; - mColorDetector = colorDetector; - } - - @Override - public void run() { - Thread thread = Thread.currentThread(); - ColorIterator.Pixel pixel = new ColorIterator.Pixel(); - while (mResultBox.isNull() && mColorIterator.hasNext() && !thread.isInterrupted()) { - mColorIterator.nextColor(pixel); - if (mColorDetector.detectsColor(pixel.red, pixel.green, pixel.blue)) { - mResultBox.set(new Point(mColorIterator.getX(), mColorIterator.getY())); - break; - } - } - if (thread.isInterrupted()) { - throw new ScriptInterruptedException(); - } - } - } - - private static class FindAllColorsRunnable implements Runnable { - - private final List mResult; - private final ColorIterator mColorIterator; - private final ColorDetector mColorDetector; - - private FindAllColorsRunnable(List result, ColorIterator colorIterator, ColorDetector colorDetector) { - mResult = result; - mColorIterator = colorIterator; - mColorDetector = colorDetector; - } - - @Override - public void run() { - Thread thread = Thread.currentThread(); - ColorIterator.Pixel pixel = new ColorIterator.Pixel(); - while (mColorIterator.hasNext() && !thread.isInterrupted()) { - mColorIterator.nextColor(pixel); - if (mColorDetector.detectsColor(pixel.red, pixel.green, pixel.blue)) { - mResult.add(new Point(mColorIterator.getX(), mColorIterator.getY())); - } - } - if (thread.isInterrupted()) { - throw new ScriptInterruptedException(); - } - } - } - } \ No newline at end of file diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ColorIterator.java b/autojs/src/main/java/com/stardust/autojs/core/image/ColorIterator.java deleted file mode 100644 index 87db8a69..00000000 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ColorIterator.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.stardust.autojs.core.image; - -import android.graphics.Rect; -import android.media.Image; - -import java.nio.ByteBuffer; - -/** - * Created by Stardust on 2017/5/20. - */ - -public interface ColorIterator { - - class Pixel { - int red; - int green; - int blue; - } - - boolean hasNext(); - - void nextColor(Pixel pixel); - - int getX(); - - int getY(); - - abstract class ImageColorIterator implements ColorIterator { - - protected final ByteBuffer mByteBuffer; - protected final Rect mIterateArea; - - public ImageColorIterator(Image image, Rect area, boolean duplicateBuffer) { - Image.Plane plane = image.getPlanes()[0]; - if (duplicateBuffer) { - mByteBuffer = plane.getBuffer().duplicate(); - } else { - mByteBuffer = plane.getBuffer(); - } - mIterateArea = area; - } - - public ImageColorIterator(Image image, Rect area) { - this(image, area, false); - } - - - protected void skip(int i) { - mByteBuffer.position(mByteBuffer.position() + i); - } - } - - class SequentialIterator extends ImageColorIterator { - - private static final String LOG_TAG = "SequentialIterator"; - - private final int mRowStride; - private final int mSkipPerRow; - private final int mWidth; - private final int mHeight; - private int mX = -1; - private int mY = 0; - - public SequentialIterator(Image image, Rect area, boolean duplicateBuffer) { - super(image, area, duplicateBuffer); - Image.Plane plane = image.getPlanes()[0]; - int pixelStride = plane.getPixelStride(); - mRowStride = plane.getRowStride(); - mWidth = area.width(); - mHeight = area.height(); - int rowPadding = mRowStride - pixelStride * image.getWidth(); - mSkipPerRow = rowPadding + (image.getWidth() - mWidth) * pixelStride; - int offset = mIterateArea.top * mRowStride + mIterateArea.left * pixelStride; - mByteBuffer.position(offset); - } - - public SequentialIterator(Image image, Rect area) { - this(image, area, false); - } - - public SequentialIterator(Image image) { - this(image, new Rect(0, 0, image.getWidth(), image.getHeight())); - } - - - @Override - public boolean hasNext() { - return mX < mWidth - 1 || mY < mHeight - 1; - } - - @Override - public int getX() { - return mIterateArea.left + mX; - } - - @Override - public int getY() { - return mIterateArea.top + mY; - } - - @Override - public void nextColor(Pixel pixel) { - if (mX == mWidth - 1) { - skip(mSkipPerRow); - mX = 0; - mY++; - } else { - mX++; - } - pixel.red = mByteBuffer.get() & 0xff; - pixel.green = mByteBuffer.get() & 0xff; - pixel.blue = mByteBuffer.get() & 0xff; - mByteBuffer.get(); - } - } - - - // TODO: 2017/5/29 中心螺旋。未完成。 - class CentralSpiralIterator extends ImageColorIterator { - - private static final int DIRECTION_RIGHT = 0; - private static final int DIRECTION_TOP = 1; - private static final int DIRECTION_LEFT = 2; - private static final int DIRECTION_BOTTOM = 3; - - private final int mPixelStride, mRowStride; - private int mNextStepSkip; - private int mStepCount; - private int mMaxStep = 1; - private int mDirection = DIRECTION_RIGHT; - - public CentralSpiralIterator(Image image, Rect area, boolean duplicateBuffer) { - super(image, area, duplicateBuffer); - Image.Plane plane = image.getPlanes()[0]; - mPixelStride = mNextStepSkip = plane.getPixelStride(); - mRowStride = plane.getRowStride(); - mByteBuffer.position(area.centerX() * mPixelStride + area.centerY() * mRowStride); - } - - public CentralSpiralIterator(Image image, Rect area) { - this(image, area, false); - } - - @Override - public boolean hasNext() { - return mByteBuffer.position() < mByteBuffer.limit(); - } - - @Override - public void nextColor(Pixel pixel) { - int c = mByteBuffer.getInt(); - skip(mNextStepSkip); - mStepCount++; - if (mStepCount == mMaxStep) { - mStepCount = 0; - mMaxStep++; - mDirection = (mDirection + 1) & 4; - switch (mDirection) { - case DIRECTION_RIGHT: - mNextStepSkip = mPixelStride; - break; - case DIRECTION_TOP: - mNextStepSkip = -mRowStride; - break; - case DIRECTION_LEFT: - mNextStepSkip = -mPixelStride; - break; - case DIRECTION_BOTTOM: - mNextStepSkip = mRowStride; - break; - } - } - } - - @Override - public int getX() { - return 0; - } - - @Override - public int getY() { - return 0; - } - } - - -} diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ConcurrentColorIterator.java b/autojs/src/main/java/com/stardust/autojs/core/image/ConcurrentColorIterator.java deleted file mode 100644 index 3a0a0a56..00000000 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ConcurrentColorIterator.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.stardust.autojs.core.image; - -import android.graphics.Rect; -import android.media.Image; - -import java.nio.ByteBuffer; - -/** - * Created by Stardust on 2017/5/21. - */ - -public interface ConcurrentColorIterator { - - void nextColor(Pixel pixel); - - class Pixel { - - public int x; - public int y; - public int color; - public boolean valid = true; - - } - - abstract class ConcurrentImageColorIterator implements ConcurrentColorIterator { - - protected final ByteBuffer mByteBuffer; - protected final int mImageWidth, mImageHeight; - protected final Rect mIterateArea; - protected final int mAreaWidth, mAreaHeight; - protected volatile int mX = -1, mY; - - public ConcurrentImageColorIterator(Image image, Rect area) { - Image.Plane plane = image.getPlanes()[0]; - mByteBuffer = plane.getBuffer(); - mImageWidth = image.getWidth(); - mImageHeight = image.getHeight(); - mIterateArea = area; - mAreaWidth = area.width(); - mAreaHeight = area.height(); - } - - - protected void skip(int i) { - mByteBuffer.position(mByteBuffer.position() + i); - } - } - - class ConcurrentSequentialIterator extends ConcurrentImageColorIterator { - - private final int mSkipPerRow; - - public ConcurrentSequentialIterator(Image image, Rect area) { - super(image, area); - Image.Plane plane = image.getPlanes()[0]; - int pixelStride = plane.getPixelStride(); - int rowStride = plane.getRowStride(); - int rowPadding = rowStride - pixelStride * mImageWidth; - mSkipPerRow = (mImageWidth - mAreaWidth) * pixelStride + rowPadding; - int offset = mIterateArea.top * rowStride + mIterateArea.left * pixelStride; - mByteBuffer.position(offset); - } - - // TODO: 2017/5/21 对锁的竞争造成并发速度极慢。能否做到无锁？ - @Override - public synchronized void nextColor(Pixel pixel) { - if (!(mY < mAreaHeight - 1 || mX < mAreaWidth - 1)) { - pixel.valid = false; - return; - } - int c = mByteBuffer.getInt(); - c = ((c & 0xff) << 16) | (c & 0xff00) | ((c & 0xff0000) >> 16) | 0xff000000; - mX++; - if (mX == mAreaWidth) { - mX = 0; - mY++; - if (mSkipPerRow > 0) { - skip(mSkipPerRow); - } - } - pixel.x = mX + mIterateArea.left; - pixel.y = mY + mIterateArea.top; - pixel.color = c; - } - } - -} diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java b/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java new file mode 100644 index 00000000..c8f25ab4 --- /dev/null +++ b/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java @@ -0,0 +1,117 @@ +package com.stardust.autojs.core.image; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Color; +import android.graphics.ImageFormat; +import android.media.Image; +import android.os.Build; +import android.support.annotation.RequiresApi; + +import com.stardust.autojs.runtime.api.Images; +import com.stardust.pio.UncheckedIOException; + +import org.opencv.android.Utils; +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfInt; +import org.opencv.highgui.Highgui; +import org.opencv.imgproc.Imgproc; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; + +/** + * Created by Stardust on 2017/11/25. + */ + +public class ImageWrapper { + + private Mat mMat; + private int mWidth; + private int mHeight; + private Bitmap mBitmap; + + public ImageWrapper(Mat mat) { + mMat = mat; + mWidth = mat.cols(); + mHeight = mat.rows(); + } + + public ImageWrapper(Bitmap bitmap) { + mBitmap = bitmap; + mWidth = bitmap.getWidth(); + mHeight = bitmap.getHeight(); + } + + + @RequiresApi(api = Build.VERSION_CODES.KITKAT) + public static ImageWrapper ofImage(Image image) { + if (image == null) { + return null; + } + return new ImageWrapper(toBitmap(image)); + } + + public static ImageWrapper ofBitmap(Bitmap bitmap) { + if (bitmap == null) { + return null; + } + return new ImageWrapper(bitmap); + } + + @RequiresApi(api = Build.VERSION_CODES.KITKAT) + public static Bitmap toBitmap(Image image) { + Image.Plane plane = image.getPlanes()[0]; + ByteBuffer buffer = plane.getBuffer(); + buffer.position(0); + int pixelStride = plane.getPixelStride(); + int rowPadding = plane.getRowStride() - pixelStride * image.getWidth(); + Bitmap bitmap = Bitmap.createBitmap(image.getWidth() + rowPadding / pixelStride, image.getHeight(), Bitmap.Config.ARGB_8888); + bitmap.copyPixelsFromBuffer(buffer); + return bitmap; + } + + public int getWidth() { + return mWidth; + } + + public int getHeight() { + return mHeight; + } + + public Mat getMat() { + if (mMat == null) { + mMat = new Mat(); + Utils.bitmapToMat(mBitmap, mMat); + } + return mMat; + } + + public void saveTo(String path) { + if (mBitmap != null) { + try { + mBitmap.compress(Bitmap.CompressFormat.PNG, 100, new FileOutputStream(path)); + } catch (FileNotFoundException e) { + throw new UncheckedIOException(e); + } + } else { + Highgui.imwrite(path, mMat); + } + } + + public int getPixel(int x, int y) { + if (mBitmap != null) { + return mBitmap.getPixel(x, y); + } + double[] channels = mMat.get(x, y); + return Color.argb((int) channels[3], (int) channels[0], (int) channels[1], (int) channels[2]); + } + + public Bitmap getBitmap() { + return mBitmap; + } +} diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ScreenCapturer.java b/autojs/src/main/java/com/stardust/autojs/core/image/ScreenCapturer.java index fd482038..95671c46 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ScreenCapturer.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/ScreenCapturer.java @@ -83,25 +83,26 @@ public class ScreenCapturer { } private void setImageListener(Handler handler) { - mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() { - @Override - public void onImageAvailable(ImageReader reader) { - if (mCachedImage != null) { - synchronized (mCachedImageLock) { - if (mCachedImage != null) { - mCachedImage.close(); - } - mCachedImage = reader.acquireLatestImage(); - return; + mImageReader.setOnImageAvailableListener(reader -> { + if (mCachedImage != null) { + synchronized (mCachedImageLock) { + if (mCachedImage != null) { + mCachedImage.close(); } + mCachedImage = reader.acquireLatestImage(); + mCachedImageLock.notify(); + return; } - mCachedImage = reader.acquireLatestImage(); } + mCachedImage = reader.acquireLatestImage(); }, handler); } @Nullable public Image capture() { + if (mUnderUsingImage == null && mCachedImage == null) { + waitForImageAvailable(); + } if (mCachedImage != null) { if (mUnderUsingImage != null) mUnderUsingImage.close(); @@ -113,6 +114,16 @@ public class ScreenCapturer { return mUnderUsingImage; } + private void waitForImageAvailable() { + synchronized (mCachedImageLock) { + try { + mCachedImageLock.wait(); + } catch (InterruptedException e) { + throw new ScriptInterruptedException(); + } + } + } + public int getScreenWidth() { return mScreenWidth; } diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java new file mode 100644 index 00000000..18556c5f --- /dev/null +++ b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java @@ -0,0 +1,182 @@ +package com.stardust.autojs.core.image; + +import android.util.Pair; +import android.util.TimingLogger; + +import com.stardust.util.Nath; + +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.Point; +import org.opencv.core.Rect; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + + +/** + * Created by Stardust on 2017/11/25. + */ + +public class TemplateMatching { + + private static final String LOG_TAG = "TemplateMatching"; + + public static final int MAX_LEVEL_AUTO = -1; + + public static Point fastTemplateMatching(Mat img, Mat template, float threshold) { + return fastTemplateMatching(img, template, Imgproc.TM_CCOEFF_NORMED, 0.75f, threshold, MAX_LEVEL_AUTO); + } + + public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel) { + TimingLogger logger = new TimingLogger(LOG_TAG, "fast_tm"); + if (maxLevel == MAX_LEVEL_AUTO) { + maxLevel = selectPyramidLevel(img, template); + logger.addSplit("selectPyramidLevel:" + maxLevel); + } + Point p = null; + Mat matchResult; + double similarity = 0; + for (int level = maxLevel; level >= 0; level--) { + Mat src = getPyramidDownAtLevel(img, level); + Mat currentTemplate = getPyramidDownAtLevel(template, level); + if (p == null) { + if (!shouldContinueMatching(level, maxLevel)) { + break; + } + matchResult = matchTemplate(src, currentTemplate, matchMethod); + Pair bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold); + p = bestMatched.first; + similarity = bestMatched.second; + } else { + Rect r = getROI(p, src, currentTemplate); + matchResult = matchTemplate(new Mat(src, r), currentTemplate, matchMethod); + Pair bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold); + if (bestMatched.second < weakThreshold) { + p = null; + break; + } + p = bestMatched.first; + similarity = bestMatched.second; + p.x += r.x; + p.y += r.y; + if (bestMatched.second >= strictThreshold) { + pyrUp(p, level); + break; + } + } + logger.addSplit("level:" + level + " point:" + p); + } + logger.addSplit("result:" + p); + logger.dumpToLog(); + if (similarity < strictThreshold) { + return null; + } + return p; + } + + private static Mat getPyramidDownAtLevel(Mat m, int level) { + if (level == 0) { + return m; + } + int cols = m.cols(); + int rows = m.rows(); + for (int i = 0; i < level; i++) { + cols = (cols + 1) / 2; + rows = (rows + 1) / 2; + } + Mat r = new Mat(rows, cols, m.type()); + Imgproc.resize(m, r, new Size(cols, rows)); + return r; + } + + private static void pyrUp(Point p, int level) { + for (int i = 0; i < level; i++) { + p.x *= 2; + p.y *= 2; + } + } + + private static boolean shouldContinueMatching(int level, int maxLevel) { + if (level == maxLevel && level != 0) { + return true; + } + if (maxLevel <= 2) { + return false; + } + return level == maxLevel - 1; + } + + private static Rect getROI(Point p, Mat src, Mat currentTemplate) { + int x = (int) (p.x * 2 - currentTemplate.rows() / 4); + x = Math.max(0, x); + int y = (int) (p.y * 2 - currentTemplate.cols() / 4); + y = Math.max(0, y); + int w = (int) (currentTemplate.rows() * 1.5); + int h = (int) (currentTemplate.cols() * 1.5); + if (x + w >= src.cols()) { + w = src.cols() - x - 1; + } + if (y + h >= src.rows()) { + h = src.rows() - y - 1; + } + return new Rect(x, y, w, h); + } + + private static int selectPyramidLevel(Mat img, Mat template) { + int minDim = Nath.min(img.rows(), img.cols(), template.rows(), template.cols()); + //这里选取12为图像缩小后的最小宽高，从而用log(2, minDim / 16)得到最多可以经过几次缩小。 + int maxLevel = (int) (Math.log(minDim / 7) / Math.log(2)); + if (maxLevel < 0) { + return 0; + } + //上限为6 + return Math.min(6, maxLevel); + } + + + public static List buildPyramid(Mat mat, int maxLevel) { + List pyramid = new ArrayList<>(); + pyramid.add(mat); + for (int i = 0; i < maxLevel; i++) { + Mat m = new Mat((mat.rows() + 1) / 2, (mat.cols() + 1) / 2, mat.type()); + Imgproc.pyrDown(mat, m); + pyramid.add(m); + mat = m; + } + return pyramid; + } + + public static Mat matchTemplate(Mat img, Mat temp, int match_method) { + int result_cols = img.cols() - temp.cols() + 1; + int result_rows = img.rows() - temp.rows() + 1; + Mat result = new Mat(result_rows, result_cols, CvType.CV_32FC1); + Imgproc.matchTemplate(img, temp, result, match_method); + return result; + } + + public static Pair getBestMatched(Mat tmResult, int matchMethod, float threshold) { + TimingLogger logger = new TimingLogger(LOG_TAG, "best_matched_point"); + // FIXME: 2017/11/26 正交化? + // Core.normalize(tmResult, tmResult, 0, 1, Core.NORM_MINMAX, -1, new Mat()); + Core.MinMaxLocResult mmr = Core.minMaxLoc(tmResult); + logger.addSplit("minMaxLoc"); + double value; + Point pos; + if (matchMethod == Imgproc.TM_SQDIFF || matchMethod == Imgproc.TM_SQDIFF_NORMED) { + pos = mmr.minLoc; + value = -mmr.minVal; + } else { + pos = mmr.maxLoc; + value = mmr.maxVal; + } + logger.addSplit("value:" + value); + logger.dumpToLog(); + return new Pair<>(pos, value); + } + + +} diff --git a/autojs/src/main/java/com/stardust/autojs/core/inputevent/RootAutomator.java b/autojs/src/main/java/com/stardust/autojs/core/inputevent/RootAutomator.java index bd5e3cfd..e69ed078 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/inputevent/RootAutomator.java +++ b/autojs/src/main/java/com/stardust/autojs/core/inputevent/RootAutomator.java @@ -164,8 +164,8 @@ public class RootAutomator { public void touchUp(int id) throws IOException { sendEvent(EV_ABS, ABS_MT_TRACKING_ID, id); - sendEvent(EV_KEY, BTN_TOUCH, 0x00000000); - sendEvent(EV_KEY, BTN_TOOL_FINGER, 0x00000000); + // sendEvent(EV_KEY, BTN_TOUCH, 0x00000000); + // sendEvent(EV_KEY, BTN_TOOL_FINGER, 0x00000000); sendEvent(EV_SYN, SYN_REPORT, 0x00000000); } diff --git a/autojs/src/main/java/com/stardust/autojs/execution/ScriptExecuteActivity.java b/autojs/src/main/java/com/stardust/autojs/execution/ScriptExecuteActivity.java index 4933e4f6..a82da30a 100644 --- a/autojs/src/main/java/com/stardust/autojs/execution/ScriptExecuteActivity.java +++ b/autojs/src/main/java/com/stardust/autojs/execution/ScriptExecuteActivity.java @@ -60,16 +60,30 @@ public class ScriptExecuteActivity extends AppCompatActivity implements Thread.U prepare(); doExecution(); } catch (Exception e) { - mExecutionListener.onException(mScriptExecution, e); - super.finish(); + onException(e); } } + private void onException(Exception e) { + mExecutionListener.onException(mScriptExecution, e); + super.finish(); + } + @SuppressWarnings("unchecked") private void doExecution() { mScriptEngine.setTag(ScriptEngine.TAG_SOURCE, mScriptSource); mExecutionListener.onStart(mScriptExecution); - mResult = mScriptEngine.execute(mScriptSource); + ((LoopBasedJavaScriptEngine) mScriptEngine).execute(mScriptSource, new LoopBasedJavaScriptEngine.ExecuteCallback() { + @Override + public void onResult(Object r) { + mResult = r; + } + + @Override + public void onException(Exception e) { + ScriptExecuteActivity.this.onException(e); + } + }); } private void prepare() { @@ -94,8 +108,7 @@ public class ScriptExecuteActivity extends AppCompatActivity implements Thread.U @Override public void uncaughtException(Thread t, Throwable e) { - mExecutionListener.onException(mScriptExecution, (Exception) e); - super.finish(); + onException((Exception) e); } private static class ActivityScriptExecution extends ScriptExecution.AbstractScriptExecution { diff --git a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java index 7bfe2523..73e22ec7 100644 --- a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java +++ b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java @@ -6,6 +6,7 @@ import android.content.Intent; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.Matrix; +import android.graphics.Point; import android.media.Image; import android.os.Build; import android.os.Handler; @@ -16,8 +17,10 @@ import android.view.Surface; import android.view.WindowManager; import com.stardust.autojs.core.image.ColorFinder; +import com.stardust.autojs.core.image.ImageWrapper; import com.stardust.autojs.core.image.ScreenCaptureRequester; import com.stardust.autojs.core.image.ScreenCapturer; +import com.stardust.autojs.core.image.TemplateMatching; import com.stardust.autojs.runtime.ScriptRuntime; import com.stardust.autojs.runtime.exception.ScriptInterruptedException; import com.stardust.autojs.annotation.ScriptVariable; @@ -25,6 +28,15 @@ import com.stardust.concurrent.VolatileBox; import com.stardust.pio.UncheckedIOException; import com.stardust.util.ScreenMetrics; +import org.opencv.android.Utils; +import org.opencv.contrib.FaceRecognizer; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.Rect; +import org.opencv.core.Scalar; +import org.opencv.highgui.Highgui; +import org.opencv.imgproc.Imgproc; + import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.nio.ByteBuffer; @@ -56,7 +68,6 @@ public class Images { @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) public boolean requestScreenCapture(final int width, final int height) { mScriptRuntime.requiresApi(21); - colorFinder.prestartThreads(); final VolatileBox requestResult = new VolatileBox<>(); mScreenCaptureRequester.setOnActivityResultCallback(new ScreenCaptureRequester.Callback() { @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) @@ -84,19 +95,17 @@ public class Images { } @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) - public Image captureScreen() { + public ImageWrapper captureScreen() { mScriptRuntime.requiresApi(21); if (mScreenCapturer == null) { throw new SecurityException("No screen capture permission"); } - colorFinder.prestartThreads(); - return mScreenCapturer.capture(); + return ImageWrapper.ofImage(mScreenCapturer.capture()); } @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) public boolean captureScreen(String path) { - mScriptRuntime.requiresApi(21); - Image image = mScreenCapturer.capture(); + ImageWrapper image = captureScreen(); if (image != null) { saveImage(image, path); return true; @@ -104,22 +113,10 @@ public class Images { return false; } - public void saveImage(Image image, String path) { - Bitmap bitmap = toBitmap(image); - saveBitmap(bitmap, path); - bitmap.recycle(); + public void saveImage(ImageWrapper image, String path) { + image.saveTo(path); } - public static Bitmap toBitmap(Image image) { - Image.Plane plane = image.getPlanes()[0]; - ByteBuffer buffer = plane.getBuffer(); - buffer.position(0); - int pixelStride = plane.getPixelStride(); - int rowPadding = plane.getRowStride() - pixelStride * image.getWidth(); - Bitmap bitmap = Bitmap.createBitmap(image.getWidth() + rowPadding / pixelStride, image.getHeight(), Bitmap.Config.ARGB_8888); - bitmap.copyPixelsFromBuffer(buffer); - return bitmap; - } public static int pixel(Image image, int x, int y) { int originX = x; @@ -134,15 +131,15 @@ public class Images { return (c & 0xff000000) + ((c & 0xff) << 16) + (c & 0x00ff00) + ((c & 0xff0000) >> 16); } - public static int pixel(Bitmap bitmap, int x, int y) { - x = ScreenMetrics.rescaleX(x, bitmap.getWidth()); - y = ScreenMetrics.rescaleY(y, bitmap.getHeight()); - return bitmap.getPixel(x, y); + public static int pixel(ImageWrapper image, int x, int y) { + x = ScreenMetrics.rescaleX(x, image.getWidth()); + y = ScreenMetrics.rescaleY(y, image.getHeight()); + return image.getPixel(x, y); } - public static Bitmap read(String path) { - return BitmapFactory.decodeFile(path); + public ImageWrapper read(String path) { + return ImageWrapper.ofBitmap(BitmapFactory.decodeFile(path)); } public static void saveBitmap(Bitmap bitmap, String path) { @@ -153,24 +150,6 @@ public class Images { } } - public static void saveBitmap(Bitmap bitmap, String path, int width, int height) { - if (width != bitmap.getWidth() || height != bitmap.getHeight()) { - Bitmap scaleBitmap = scaleBitmap(bitmap, width, height); - saveBitmap(scaleBitmap, path); - if (scaleBitmap != bitmap) { - scaleBitmap.recycle(); - } - } else { - saveBitmap(bitmap, path); - } - } - - public void saveImage(Image image, String path, int width, int height) { - Bitmap bitmap = toBitmap(image); - saveBitmap(bitmap, path, width, height); - bitmap.recycle(); - } - public static Bitmap scaleBitmap(Bitmap origin, int newWidth, int newHeight) { if (origin == null) { return null; @@ -190,4 +169,37 @@ public class Images { } } + public Point findImage(ImageWrapper image, ImageWrapper template) { + return findImage(image, template, 0.9f, null); + } + + public Point findImage(ImageWrapper image, ImageWrapper template, float threshold) { + return findImage(image, template, threshold, null); + } + + public Point findImage(ImageWrapper image, ImageWrapper template, float threshold, Rect rect) { + return findImage(image, template, threshold, rect, TemplateMatching.MAX_LEVEL_AUTO); + } + + public Point findImage(ImageWrapper image, ImageWrapper template, float threshold, Rect rect, int maxLevel) { + Mat src = image.getMat(); + if (rect != null) { + src = new Mat(src, rect); + } + org.opencv.core.Point point = TemplateMatching.fastTemplateMatching(src, template.getMat(), threshold); + if (point != null && rect != null) { + point.x += rect.x; + point.y += rect.y; + } + return toAndroidPoint(point); + } + + + public static Point toAndroidPoint(org.opencv.core.Point p) { + if (p == null) { + return null; + } + return new Point((int) p.x, (int) p.y); + } + } diff --git a/autojs/src/main/java/com/stardust/autojs/runtime/exception/ScriptInterruptedException.java b/autojs/src/main/java/com/stardust/autojs/runtime/exception/ScriptInterruptedException.java index 28ef741e..37d64c37 100644 --- a/autojs/src/main/java/com/stardust/autojs/runtime/exception/ScriptInterruptedException.java +++ b/autojs/src/main/java/com/stardust/autojs/runtime/exception/ScriptInterruptedException.java @@ -1,12 +1,13 @@ package com.stardust.autojs.runtime.exception; + /** * Created by Stardust on 2017/4/30. */ public class ScriptInterruptedException extends ScriptException { - public ScriptInterruptedException(){ + public ScriptInterruptedException() { } @@ -15,7 +16,13 @@ public class ScriptInterruptedException extends ScriptException { } public static boolean causedByInterrupted(Throwable e) { - return e instanceof ScriptInterruptedException || e.getCause() instanceof ScriptInterruptedException; + while (e != null) { + if (e instanceof ScriptInterruptedException || e instanceof InterruptedException) { + return true; + } + e = e.getCause(); + } + return false; } } diff --git a/automator/src/main/java/com/stardust/notification/Notification.java b/automator/src/main/java/com/stardust/notification/Notification.java index ecb388db..4a523d23 100644 --- a/automator/src/main/java/com/stardust/notification/Notification.java +++ b/automator/src/main/java/com/stardust/notification/Notification.java @@ -57,9 +57,9 @@ public class Notification extends android.app.Notification { public String toString() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { return "Notification{" + - "packageName='" + mPackageName + '\'' + - "title = " + getTitle() + '\'' + - "text='" + getText() + '\'' + + "packageName='" + mPackageName + "', " + + "title='" + getTitle() + ", " + + "text='" + getText() + "'" + "} "; } return super.toString(); diff --git a/common/src/main/java/com/stardust/util/IntentExtras.java b/common/src/main/java/com/stardust/util/IntentExtras.java index cf70fff4..e3a65ec0 100644 --- a/common/src/main/java/com/stardust/util/IntentExtras.java +++ b/common/src/main/java/com/stardust/util/IntentExtras.java @@ -43,6 +43,7 @@ public class IntentExtras implements Serializable { private IntentExtras(int id) { mMap = extraStore.get(id); + extraStore.remove(id); mMaxId = id; } diff --git a/common/src/main/java/com/stardust/util/Nath.java b/common/src/main/java/com/stardust/util/Nath.java new file mode 100644 index 00000000..b5f023d2 --- /dev/null +++ b/common/src/main/java/com/stardust/util/Nath.java @@ -0,0 +1,16 @@ +package com.stardust.util; + +/** + * Created by Stardust on 2017/11/26. + */ + +public class Nath { + + public static int min(int... ints) { + int min = ints[0]; + for (int i = 1; i < ints.length; i++) { + min = ints[i] < min ? ints[i] : min; + } + return min; + } +}