tt/AutoJs6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

6.6.3 - Alpha4 - 新增 images.matchFeatures/detectAndComputeFeatures 方法 (issue #366)

Browse Source
This commit is contained in:
SuperMonster003
2025-05-14 23:59:13 +08:00
parent 514d39eb6c
commit d3a9cd0f26
25 changed files with 535 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/src/main/assets-app/doc/CHANGELOG-zh-Hans.md
View File

@@ -6,12 +6,13 @@
# v6.6.3
###### 2025/05/13
###### 2025/05/14
* `新增` 版本历史功能, 可查看发行版本历史更新记录 (多语言) 与统计数据
* `新增` timers.keepAlive 方法 (已全局化), 用于保持脚本活跃状态
* `新增` engines.on('start/stop/error', callback) 等事件监听方法, 用于监听脚本引擎全局事件
* `新增` images.detectMultiColors 方法, 用于多点颜色校验 _[`issue #374`](http://issues.autojs6.com/374)_
* `新增` images.matchFeatures/detectAndComputeFeatures 方法, 支持全分辨率找图 (Ref to [Auto.js Pro](https://g.pro.autojs.org/)) _[`issue #366`](http://issues.autojs6.com/366)_
* `修复` 主页文档标签显示在线文档时部分内容被系统导航栏遮挡的问题
* `修复` 部分设备代码编辑器空行显示方框字符的问题
* `修复` 主题色设置页面调色盘对话框可能无限叠加的问题
@@ -25,7 +26,7 @@
* `修复` console.setContentBackgroundColor 方法无法接受颜色名称参数的问题 _[`issue #384`](http://issues.autojs6.com/384)_
* `修复` README.md 中部分语言日期格式不正确的问题
* `优化` 布局分析支持控件隐藏 (by [TonyJiangWJ](https://github.com/TonyJiangWJ)) _[`pr #371`](http://pr.autojs6.com/371)_ _[`issue #355`](http://issues.autojs6.com/355)_
* `优化` 布局分析菜单使用渐变分隔线实现一定程度的功能分组
* `优化` 布局分析菜单添加渐变分隔线实现一定程度的功能分组
* `优化` 主题色扩充适配范围并支持更多控件类型
* `优化` 主页抽屉在横向屏幕或超宽屏幕的宽度适应性
* `优化` 关于应用与开发者页面增加水平布局及小屏布局适配

5
app/src/main/assets-app/doc/CHANGELOG.md
View File

@@ -6,12 +6,13 @@
# v6.6.3
###### 2025/05/13
###### 2025/05/14
* `新增` 版本历史功能, 可查看发行版本历史更新记录 (多语言) 与统计数据
* `新增` timers.keepAlive 方法 (已全局化), 用于保持脚本活跃状态
* `新增` engines.on('start/stop/error', callback) 等事件监听方法, 用于监听脚本引擎全局事件
* `新增` images.detectMultiColors 方法, 用于多点颜色校验 _[`issue #374`](http://issues.autojs6.com/374)_
* `新增` images.matchFeatures/detectAndComputeFeatures 方法, 支持全分辨率找图 (Ref to [Auto.js Pro](https://g.pro.autojs.org/)) _[`issue #366`](http://issues.autojs6.com/366)_
* `修复` 主页文档标签显示在线文档时部分内容被系统导航栏遮挡的问题
* `修复` 部分设备代码编辑器空行显示方框字符的问题
* `修复` 主题色设置页面调色盘对话框可能无限叠加的问题
@@ -25,7 +26,7 @@
* `修复` console.setContentBackgroundColor 方法无法接受颜色名称参数的问题 _[`issue #384`](http://issues.autojs6.com/384)_
* `修复` README.md 中部分语言日期格式不正确的问题
* `优化` 布局分析支持控件隐藏 (by [TonyJiangWJ](https://github.com/TonyJiangWJ)) _[`pr #371`](http://pr.autojs6.com/371)_ _[`issue #355`](http://issues.autojs6.com/355)_
* `优化` 布局分析菜单使用渐变分隔线实现一定程度的功能分组
* `优化` 布局分析菜单添加渐变分隔线实现一定程度的功能分组
* `优化` 主题色扩充适配范围并支持更多控件类型
* `优化` 主页抽屉在横向屏幕或超宽屏幕的宽度适应性
* `优化` 关于应用与开发者页面增加水平布局及小屏布局适配

98
app/src/main/java/org/autojs/autojs/core/automator/UiObject.kt
View File

@@ -19,6 +19,7 @@ import org.autojs.autojs.runtime.ScriptRuntime
import org.autojs.autojs.util.DisplayUtils.toRoundIntX
import org.autojs.autojs.util.DisplayUtils.toRoundIntY
import org.autojs.autojs.util.RhinoUtils
import org.autojs.autojs.util.StringUtils
import org.autojs.autojs.util.StringUtils.str
import org.autojs.autojs6.R
import org.mozilla.javascript.BaseFunction
@@ -436,70 +437,55 @@ open class UiObject(
}
}
fun summary(): String {
val separatorLv0 = "\n"
val separatorLv1 = "$separatorLv0 "
val separatorLv2 = "$separatorLv1 "
val dataList = listOf<Pair<String, () -> Any?>>(
fun summary(): String = listOf(
/* Common */
/* Common */
"packageName" to { packageName() },
"parent" to { parent?.className },
"id" to { id() },
"fullId" to { fullId() },
"idHex" to { idHex() },
"desc" to { desc() },
"text" to { text() },
"bounds" to { bounds() },
"center" to { center() },
"className" to { className() },
"clickable" to { clickable() },
"longClickable" to { longClickable() },
"scrollable" to { scrollable() },
"indexInParent" to { indexInParent() },
"childCount" to { childCount() },
"depth" to { depth() },
"packageName" to { packageName() },
"parent" to { parent?.className },
"id" to { id() },
"fullId" to { fullId() },
"idHex" to { idHex() },
"desc" to { desc() },
"text" to { text() },
"bounds" to { bounds() },
"center" to { center() },
"className" to { className() },
"clickable" to { clickable() },
"longClickable" to { longClickable() },
"scrollable" to { scrollable() },
"indexInParent" to { indexInParent() },
"childCount" to { childCount() },
"depth" to { depth() },
/* Regular */
/* Regular */
"checked" to { checked() },
"enabled" to { enabled() },
"editable" to { editable() },
"focusable" to { focusable() },
"checkable" to { checkable() },
"selected" to { selected() },
"dismissable" to { isDismissable },
"visibleToUser" to { visibleToUser() },
"checked" to { checked() },
"enabled" to { enabled() },
"editable" to { editable() },
"focusable" to { focusable() },
"checkable" to { checkable() },
"selected" to { selected() },
"dismissable" to { isDismissable },
"visibleToUser" to { visibleToUser() },
/* Rare */
/* Rare */
"contextClickable" to { isContextClickable },
"focused" to { focused() },
"accessibilityFocused" to { isAccessibilityFocused },
"rowCount" to { rowCount() },
"columnCount" to { columnCount() },
"row" to { row() },
"column" to { column() },
"rowSpan" to { rowSpan() },
"columnSpan" to { columnSpan() },
"drawingOrder" to { drawingOrder },
"contextClickable" to { isContextClickable },
"focused" to { focused() },
"accessibilityFocused" to { isAccessibilityFocused },
"rowCount" to { rowCount() },
"columnCount" to { columnCount() },
"row" to { row() },
"column" to { column() },
"rowSpan" to { rowSpan() },
"columnSpan" to { columnSpan() },
"drawingOrder" to { drawingOrder },
/* List */
/* List */
"actions" to { actionNames() },
)
return dataList.joinToString(prefix = "{$separatorLv1", separator = separatorLv1, postfix = "$separatorLv0}") { (name, action) ->
val value = when (val actionResult = action()) {
is CharSequence -> "\"$actionResult\""
is Iterable<*> -> actionResult.joinToString(prefix = "[$separatorLv2", separator = separatorLv2, postfix = "$separatorLv1]")
is Array<*> -> actionResult.joinToString(prefix = "[$separatorLv2", separator = separatorLv2, postfix = "$separatorLv1]")
else -> actionResult
}
"$name=$value"
}
}
"actions" to { actionNames() },
).let { StringUtils.toFormattedSummary(it) }
override fun toString(): String {
val simpledClassName = "$className".substringAfterLast(".")

355
app/src/main/java/org/autojs/autojs/runtime/api/ImageFeatureMatching.java
View File

@@ -8,7 +8,9 @@ import org.autojs.autojs.core.cleaner.Cleaner;
import org.autojs.autojs.core.cleaner.ICleaner;
import org.autojs.autojs.core.ref.MonitorResource;
import org.autojs.autojs.core.ref.NativeObjectReference;
import org.autojs.autojs.util.ImageUtils;
import org.opencv.calib3d.Calib3d;
import org.opencv.core.Core;
import org.opencv.core.DMatch;
import org.opencv.core.KeyPoint;
import org.opencv.core.Mat;
@@ -26,9 +28,10 @@ import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
/**
* Created by SuperMonster003 on Jan 7, 2024.
@@ -44,122 +47,261 @@ public final class ImageFeatureMatching {
@ScriptInterface
public static int FEATURE_MATCHING_METHOD_ORB = 2;
// FIXME by SuperMonster003 on Nov 23, 2024.
// ! This function needs to be corrected or improved.
// ! zh-CN: 此函数功能需纠正或完善.
public static FeatureMatchingDescriptor createFeatureMatchingDescriptor(Mat mat, int cvtColorFlag, float scale, int method) {
if (mat == null || mat.empty()) {
throw new IllegalArgumentException("Input Mat cannot be null or empty");
// @Hint by SuperMonster003 on May 14, 2025.
// ! This method was corrected and improved by JetBrains AI Assistant.
// ! zh-CN: 此方法由 JetBrains AI Assistant 纠正并完善.
/**
* Pre-computes a feature-matching descriptor for an image. The pipeline:
* <ol>
* <li>Optional color-space conversion controlled by {@code cvtColorFlag}</li>
* <li>Optional uniform scaling specified by {@code scale}</li>
* <li>Key-point detection and descriptor extraction according to {@code method}
* (SIFT / ORB)</li>
* <li>Wrap everything into a {@link FeatureMatchingDescriptor} so that it can be
* reused later by {@link #featureMatching}</li>
* </ol></p>
*
* <p><b>zh-CN</b><br>
*
* 创建 (预计算) 特征匹配描述符.<br>
* 典型流程: <pre>
* 1. (可选) 颜色空间转换 {@code cvtColorFlag}
* 2. (可选) 缩放 {@code scale}
* 3. 依据 {@code method} (SIFT / ORB) 检测关键点并计算描述子
* 4. 封装为 {@link FeatureMatchingDescriptor} 供后续匹配复用
* </pre>
*
* <p>典型使用场景: 模板图/场景图分别调用一次当前方法, 随后在多次 {@link #featureMatching} 中复用, 以避免重复提特征带来的性能损耗.</p>
*
* @param src The source Mat. The content will not be modified.<br>
* zh-CN: 原始 {@link Mat}. 输入图像内容不会被修改.
* @param cvtColorFlag OpenCV cvtColor flag; -1 means "no conversion".<br>
* zh-CN: OpenCV 颜色空间转换标志; 传 -1 表示跳过转换.
* @param scale Scale factor; &gt;0 to resize, ≤0 or 1 to keep original size.<br>
* zh-CN: 缩放因子; >0 表示按比例缩放, ≤0 或 1 表示保持原尺寸.
* @param method Extraction method, see {@code FEATURE_MATCHING_METHOD_*}.<br>
* zh-CN: 特征提取方法; 取值见 {@code FEATURE_MATCHING_METHOD_*}.
*
* @return The generated descriptor. Release via
* {@link FeatureMatchingDescriptor#release()} or rely on GC.<br>
* zh-CN: 生成的 {@link FeatureMatchingDescriptor}; 仅当显式调用
* {@link FeatureMatchingDescriptor#release()} 或对象被 GC 时才会释放其底层资源.
*
* @throws IllegalArgumentException Thrown if {@code src} is empty or {@code method} is unsupported.<br>
* zh-CN: 当 {@code src} 为空或不支持的 {@code method} 时抛出.
*/
@ScriptInterface
public static FeatureMatchingDescriptor createFeatureMatchingDescriptor(
@NonNull Mat src,
int cvtColorFlag,
float scale,
int method
) {
if (src.empty()) {
throw new IllegalArgumentException("Input Mat is null or empty.");
}
// Add this part to ensure the image is loaded correctly
if (mat.empty()) {
throw new RuntimeException("Failed to load image. The image file might be missing or the path is incorrect.");
// Color space conversion (zh-CN: 颜色空间转换)
Mat processed = src;
if (cvtColorFlag >= 0) {
processed = new Mat();
Imgproc.cvtColor(src, processed, cvtColorFlag);
}
// Convert color if needed
Mat convertedMat = new Mat();
if (cvtColorFlag != -1) {
Imgproc.cvtColor(mat, convertedMat, cvtColorFlag);
} else {
convertedMat = mat.clone();
// Scaling (zh-CN: 缩放)
if (scale > 0f && Math.abs(scale - 1f) > 1e-3) {
Size newSize = new Size(processed.cols() * scale, processed.rows() * scale);
Mat resized = new Mat();
Imgproc.resize(processed, resized, newSize);
processed = resized;
}
// Resize the image if scaling is required
Mat scaledMat = new Mat();
if (scale > 0 && scale != 1.0f) {
Imgproc.resize(convertedMat, scaledMat, new Size(mat.cols() * scale, mat.rows() * scale));
} else {
scaledMat = convertedMat.clone();
}
// Select feature detector and descriptor extractor based on method
// Choose detector (zh-CN: 选择算子)
Feature2D detector;
if (method == FEATURE_MATCHING_METHOD_SIFT) {
detector = SIFT.create();
} else if (method == FEATURE_MATCHING_METHOD_ORB) {
detector = ORB.create();
detector = ORB.create(
/* nFeatures = */ 5000, /* 500 -> 5000, increase feature count (zh-CN: 增加特征数量) */
/* scaleFactor = */ 1.2f,
/* nLevels = */ 8,
/* edgeThreshold = */ 31,
/* firstLevel = */ 0,
/* WTA_K = */ 4, /* 2 -> 4, produces 256 bit descriptors, increases discrimination (zh-CN: 产生 256 bit 描述子, 增加区分度) */
/* scoreType = */ ORB.HARRIS_SCORE,
/* patchSize = */ 31,
/* fastThreshold = */ 20);
} else {
throw new IllegalArgumentException("Unsupported feature matching method: " + method);
}
// Detect keypoints and compute descriptors
MatOfKeyPoint keyPoints = new MatOfKeyPoint();
// Calculate KeyPoint & Descriptor (zh-CN: 计算 KeyPoint & Descriptor)
MatOfKeyPoint kps = new MatOfKeyPoint();
Mat descriptors = new Mat();
detector.detect(scaledMat, keyPoints);
detector.compute(scaledMat, keyPoints, descriptors);
detector.detectAndCompute(processed, new Mat(), kps, descriptors);
return new FeatureMatchingDescriptor(descriptors, keyPoints);
// Build reference image corner points (zh-CN: 构建参考图像的四角点)
int w = src.width();
int h = src.height();
MatOfPoint2f corners = new MatOfPoint2f(
new Point(0, 0),
new Point(w, 0),
new Point(0, h),
new Point(w, h)
);
return new FeatureMatchingDescriptor(descriptors, kps, corners);
}
// FIXME by SuperMonster003 on Nov 23, 2024.
// ! This function needs to be corrected or improved.
// ! zh-CN: 此函数功能需纠正或完善.
// @Hint by SuperMonster003 on May 14, 2025.
// ! This method was corrected and improved by JetBrains AI Assistant.
// ! zh-CN: 此方法由 JetBrains AI Assistant 纠正并完善.
/**
* It matches two pre-computed feature descriptors and optionally estimates a homography
* to locate where the <i>object</i> image appears inside the <i>scene</i> image.<br>
*
* <p>Internal steps: <br>
* <ol>
* <li>Create a suitable {@link org.opencv.features2d.DescriptorMatcher} based on {@code matcherType}</li>
* <li>Perform KNN matching (k=2) and filter using Lowe's ratio test with threshold {@code threshold}</li>
* <li>If good matches ≥4, use {@code Calib3d.findHomography(..., RANSAC)} to further eliminate outliers
* while calculating projection (quad) of object image corners in scene image</li>
* <li>If caller sets {@code debugMatchesImagePath}, save visualization of match lines to that path</li>
* </ol></p>
*
* <p><b>zh-CN</b><br>
*
* 在两张图的特征描述符之间执行匹配, 并 (可选) 估算单应矩阵以获得被 object 图在 scene 图中的投影区域.<br>
* <p>内部流程: <br>
* <ol>
* <li>根据 {@code matcherType} 创建合适的 {@link org.opencv.features2d.DescriptorMatcher}</li>
* <li>执行 KNN 匹配 (k=2) 并用 Lowe 比例测试 (阈值为 {@code threshold}) 过滤</li>
* <li>若 good matches ≥4, 则通过 {@code Calib3d.findHomography(..., RANSAC)} 进一步剔除离群点,
* 同时计算 object 图四角在 scene 图中的投影 (quad)</li>
* <li>如调用者设置了 {@code debugMatchesImagePath}, 会将匹配连线可视化保存到该路径</li>
* </ol></p>
*
* @param sceneDesc Descriptor for the scene image.<br>
* zh-CN: 目标 / 场景图的描述符.
* @param objectDesc Descriptor for the object image.<br>
* zh-CN: 模板 / 待检测对象图的描述符.
* @param matcherType Type constant of DescriptorMatcher.<br>
* zh-CN: {@link org.opencv.features2d.DescriptorMatcher} 的类型常量.
* @param debugMatchesImagePath Optional path to save a debug image showing matches; {@code null} to skip.<br>
* zh-CN: 可选调试路径; 非空时将匹配结果绘制到该文件.
* @param threshold Lowe ratio threshold (0,1); higher = looser filtering.<br>
* zh-CN: Lowe Ratio 测试阈值 (0,1). 值越大匹配越宽松.
*
* @return {@link FeatureMatchingResult}: <br>
* • {@code getPoints()} - Filtered matching point pairs (in scene coordinates)<br>
* • {@code getQuad()} - 4-point quadrilateral if homography estimation succeeded, otherwise {@code null}<br>
* • {@code getMatches()} - Rendered Mat if debug enabled, otherwise {@code null}<br>
* • zh-CN:<br>
* • {@code getPoints()} - 过滤后的匹配点对 (scene 坐标系)<br>
* • {@code getQuad()} - 若成功估算单应矩阵则为 4 点四边形, 否则为 {@code null}<br>
* • {@code getMatches()} - 若开启调试则为绘制后的 Mat, 否则为 {@code null}<br>
*
* @throws IllegalStateException Thrown if descriptors are incompatible or already released.<br>
* zh-CN: 当两侧描述子维度不兼容或资源已释放时抛出.
*/
@ScriptInterface
public static FeatureMatchingResult featureMatching(
FeatureMatchingDescriptor sceneDescriptor,
FeatureMatchingDescriptor objectDescriptor,
@NonNull FeatureMatchingDescriptor sceneDesc,
@NonNull FeatureMatchingDescriptor objectDesc,
int matcherType,
@Nullable String matchesImageToDrawPath,
@Nullable String debugMatchesImagePath,
float threshold
) {
// Step 1: Extract key points and descriptors
Mat sceneDescriptors = sceneDescriptor.getDescriptors();
Mat objectDescriptors = objectDescriptor.getDescriptors();
MatOfKeyPoint sceneKeyPoints = sceneDescriptor.getKeyPoint();
MatOfKeyPoint objectKeyPoints = objectDescriptor.getKeyPoint();
List<MatOfDMatch> knnMatches = new LinkedList<>();
DescriptorMatcher matcher = DescriptorMatcher.create(matcherType);
List<MatOfDMatch> knnMatches = new ArrayList<>();
// @Reference to Sakura小败狗 (https://blog.csdn.net/qq_42670220) by SuperMonster003 on Feb 26, 2024.
// ! https://blog.csdn.net/qq_42670220/article/details/108623752
// !
// ! knnMatch method finds the best matches in given feature descriptor sets.
// ! Using KNN-matching algorithm with k = 2, each match gets 2 closest descriptors,
// ! keeps match as final when ratio of closest distance to second closest is greater than threshold.
// !
// ! zh-CN:
// !
// ! knnMatch 方法, 在给定特征描述集合中寻找最佳匹配.
// ! 使用 KNN-matching 算法, k = 2, 每个 match 得到 2 个最接近的 descriptor,
// ! 最接近距离和次接近距离的比值大于既定值时, 作为最终 match.
matcher.knnMatch(sceneDescriptors, objectDescriptors, knnMatches, 2);
LinkedList<DMatch> niceMatches = new LinkedList<>();
matcher.knnMatch(objectDesc.getDescriptors(), sceneDesc.getDescriptors(), knnMatches, 2);
// Lowe's ratio test
// zh-CN: Lowe 比例测试
List<DMatch> goodMatches = new ArrayList<>();
for (MatOfDMatch matOfDMatch : knnMatches) {
DMatch[] matches = matOfDMatch.toArray();
if (matches.length < 2) {
continue;
}
if (matches[0].distance < threshold * matches[1].distance) {
niceMatches.add(matches[0]);
goodMatches.add(matches[0]);
}
}
List<KeyPoint> sceneKeyPointsList = sceneKeyPoints.toList();
List<Point> scenePoints = new ArrayList<>(niceMatches.size());
for (DMatch match : niceMatches) {
scenePoints.add(sceneKeyPointsList.get(match.queryIdx).pt);
// Draw matches visualization
// zh-CN: 绘制匹配图
Mat matchesImg = null;
if (debugMatchesImagePath != null && !debugMatchesImagePath.isEmpty()) {
matchesImg = new Mat();
Features2d.drawMatches(
/* img1 = */ ImageUtils.to8UC3(objectDesc.getDescriptors()), objectDesc.getKeyPoint(),
/* img2 = */ ImageUtils.to8UC3(sceneDesc.getDescriptors()), sceneDesc.getKeyPoint(),
new MatOfDMatch(goodMatches.toArray(new DMatch[0])),
matchesImg);
Imgcodecs.imwrite(debugMatchesImagePath, matchesImg);
}
// Optional: Draw matches
// if (matchesImageToDrawPath != null && !matchesImageToDrawPath.isEmpty()) {
// Mat imgMatches = new Mat();
// Features2d.drawMatches(new Mat(), sceneKeyPoints, new Mat(), objectKeyPoints, new MatOfDMatch(niceMatches.toArray(new DMatch[0])), imgMatches);
// Imgcodecs.imwrite(matchesImageToDrawPath, imgMatches);
// }
// Extract matched object / scene coordinates
// zh-CN: 提取配对的 object / scene 坐标
List<Point> objPts = new ArrayList<>();
List<Point> scenePts = new ArrayList<>();
// Step 2: Compute homography if any good matches found
Mat homography = null;
if (!niceMatches.isEmpty()) {
MatOfPoint2f obj = new MatOfPoint2f();
MatOfPoint2f scene = new MatOfPoint2f();
KeyPoint[] objKpsArr = objectDesc.getKeyPoint().toArray();
KeyPoint[] sceneKpsArr = sceneDesc.getKeyPoint().toArray();
List<Point> objectPoints = new ArrayList<>(niceMatches.size());
for (DMatch match : niceMatches) {
objectPoints.add(sceneKeyPointsList.get(match.trainIdx).pt);
for (DMatch gm : goodMatches) {
objPts.add(objKpsArr[gm.queryIdx].pt);
scenePts.add(sceneKpsArr[gm.trainIdx].pt);
}
// Calculate homography matrix (requires at least 4 matched coordinate pairs)
// zh-CN: 计算单应矩阵 (需至少 4 个配对的坐标点)
List<Point> quad = null;
Log.d(TAG, "objPts: " + objPts.size() + ", scenePts: " + scenePts.size() + ", goodMatches: " + goodMatches.size());
if (objPts.size() >= 4) {
MatOfPoint2f objMat = new MatOfPoint2f();
MatOfPoint2f sceneMat = new MatOfPoint2f();
objMat.fromList(objPts);
sceneMat.fromList(scenePts);
Mat H = Calib3d.findHomography(objMat, sceneMat, Calib3d.RANSAC, 3);
if (!H.empty()) {
MatOfPoint2f objCorners = objectDesc.getCorners();
MatOfPoint2f sceneCorners = new MatOfPoint2f();
Core.perspectiveTransform(objCorners, sceneCorners, H);
quad = sortClockwise(sceneCorners.toList());
}
obj.fromList(objectPoints);
scene.fromList(scenePoints);
homography = Calib3d.findHomography(obj, scene, Calib3d.RANSAC, 3);
}
return new FeatureMatchingResult(scenePoints, homography);
return new FeatureMatchingResult(scenePts, quad, matchesImg);
}
// Sort four points clockwise as TL, TR, BL, BR
// zh-CN: 四点按 TL, TR, BL, BR 顺时针排序
private static List<Point> sortClockwise(List<Point> pts) {
if (pts.size() != 4) return pts;
// Use centroid as reference (zh-CN: 以质心为参考)
double cx = pts.stream().mapToDouble(p -> p.x).average().orElse(0);
double cy = pts.stream().mapToDouble(p -> p.y).average().orElse(0);
// Sort by polar angle (counter-clockwise), then manually adjust to start from TL
// zh-CN: 按极角排序 (逆时针), 然后手动调到 TL 开头
Stream<Point> sorted = pts.stream()
.sorted(Comparator.comparingDouble(a -> Math.atan2(a.y - cy, a.x - cx)));
return List.of(sorted.toArray(Point[]::new));
}
private static void releaseFeatureMatchingDescriptor(long pointer) {
@@ -168,17 +310,40 @@ public final class ImageFeatureMatching {
mat.release();
}
/**
* A reusable bundle of feature-extraction results for a single image.
* zh-CN: 单张图片的特征提取结果集合.
*/
public static class FeatureMatchingDescriptor implements MonitorResource {
private final Mat mDescriptors;
private long mNativePtr;
private NativeObjectReference<MonitorResource> mRef;
private final MatOfKeyPoint mKeyPoint;
private final MatOfPoint2f mCorners;
public FeatureMatchingDescriptor(Mat descriptors, MatOfKeyPoint keyPoint) {
/**
* Constructs a descriptor object. Usually created internally by
* {@link #createFeatureMatchingDescriptor(Mat, int, float, int)}.
* Encapsulates feature detection results (keypoints and descriptors) of a single image
* for later matching operations.
* <p>
* <b>zh-CN</b><br>
* 构造一个描述符对象. 通常由 {@link #createFeatureMatchingDescriptor(Mat, int, float, int)}
* 内部调用创建. 封装单个图像的特征检测结果 (关键点和描述子) 以供后续匹配.
*
* @param descriptors Feature descriptor matrix computed by feature detector.<br>
* zh-CN: 特征检测器计算出的描述子矩阵.
* @param keyPoint Detected key points in the image.<br>
* zh-CN: 在图像中检测到的关键点集.
* @param corners Virtual corners of the image after scale, always [TL, TR, BL, BR]. Used for homography calculation.<br>
* zh-CN: 缩放后图像的虚拟四角点坐标, 按 [左上,右上,左下,右下] 顺序, 用于单应矩阵计算.
*/
public FeatureMatchingDescriptor(Mat descriptors, MatOfKeyPoint keyPoint, MatOfPoint2f corners) {
mDescriptors = descriptors;
mNativePtr = descriptors.nativeObj;
mKeyPoint = keyPoint;
mCorners = corners;
Cleaner.instance.cleanup(this, SelfCleaner.INSTANCE);
}
@@ -220,6 +385,10 @@ public final class ImageFeatureMatching {
mNativePtr = nativePtr;
}
public MatOfPoint2f getCorners() {
return mCorners;
}
public static class SelfCleaner implements ICleaner {
public static SelfCleaner INSTANCE;
@@ -239,20 +408,48 @@ public final class ImageFeatureMatching {
}
}
/**
* Immutable container holding the output of a single {@link #featureMatching} operation.
* <p>
* Depending on whether homography estimation succeeds, the {@code quad} field may be {@code null}.
* The {@code matches} image is only generated when the caller passes a non-null debug path.
* <p>
* <b>zh-CN</b><br>
* 表示一次 {@link #featureMatching} 调用结果的不可变对象.
* 若单应矩阵估算失败, {@code quad} 为 {@code null}.
* 若未开启 debug 输出, {@code matches} 为 {@code null}.
*/
public static class FeatureMatchingResult {
private final List<Point> mPoints;
private final List<Point> mQuad;
private final Mat mMatches;
public FeatureMatchingResult(List<Point> points, @Nullable Mat matches) {
mPoints = points;
/**
* @param pts Inlier points in scene image after ratio test and RANSAC.<br>
* zh-CN: 终态内点 (场景座标系).
* @param quad Projected quadrilateral of the object image in scene image.<br>
* zh-CN: 物体投影四边形.
* @param matches Debug visualization image.<br>
* zh-CN: 匹配连接图 (仅 debug 时生成).
*/
public FeatureMatchingResult(@NonNull List<Point> pts, @Nullable List<Point> quad, @Nullable Mat matches) {
mPoints = pts;
mQuad = quad;
mMatches = matches;
}
@Nullable
public Mat getMatches() {
return mMatches;
return mMatches != null ? ImageUtils.to8UC3(mMatches) : null;
}
@Nullable
public List<Point> getQuad() {
return mQuad;
}
@NonNull
public List<Point> getPoints() {
return mPoints;
}
@@ -260,8 +457,8 @@ public final class ImageFeatureMatching {
@Override
public boolean equals(Object o) {
return o == this || o instanceof FeatureMatchingResult featureMatchingResult
&& Objects.equals(mPoints, featureMatchingResult.mPoints)
&& Objects.equals(mMatches, featureMatchingResult.mMatches);
&& Objects.equals(mPoints, featureMatchingResult.mPoints)
&& Objects.equals(mMatches, featureMatchingResult.mMatches);
}
@Override

7
app/src/main/java/org/autojs/autojs/runtime/api/augment/Augmentable.kt
View File

@@ -435,8 +435,13 @@ abstract class Augmentable(private val scriptRuntime: ScriptRuntime? = null) : F
val message = globalContext.getString(R.string.error_failed_to_invoke_method_with_description, methodDescription)
val niceMessage = when (val errMsg = e.message) {
null -> message
else -> "$message. ${errMsg.replaceFirst(Regex("^(Wrapped )?\\w*(\\.\\w+)*(Exception|Error): "), "")}"
else -> {
val refined = errMsg.replaceFirst(Regex("^(Wrapped )?\\w*(\\.\\w+)*(Exception|Error): "), "")
val trailingDot = if (refined.endsWith(".")) "" else "."
"$message. $refined$trailingDot\n$e"
}
}
e.printStackTrace()
when (e) {
is WrappedIllegalArgumentException -> {
// @Hint by SuperMonster003 on Oct 31, 2024.

64
app/src/main/java/org/autojs/autojs/runtime/api/augment/images/Images.kt
View File

@@ -3,6 +3,7 @@ package org.autojs.autojs.runtime.api.augment.images
import android.annotation.SuppressLint
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.util.Log
import android.view.Gravity
import org.autojs.autojs.annotation.RhinoRuntimeFunctionInterface
import org.autojs.autojs.core.image.ColorDetector
@@ -43,6 +44,7 @@ import org.mozilla.javascript.BaseFunction
import org.mozilla.javascript.NativeArray
import org.mozilla.javascript.NativeObject
import org.mozilla.javascript.ScriptableObject
import org.opencv.core.CvType
import org.opencv.features2d.DescriptorMatcher
import org.opencv.imgproc.Imgproc
import java.io.ByteArrayOutputStream
@@ -52,9 +54,12 @@ import kotlin.collections.component3
import kotlin.collections.contains
import kotlin.math.floor
import kotlin.math.ln
import kotlin.math.max
import kotlin.math.min
import kotlin.math.pow
import kotlin.math.round
import kotlin.math.roundToInt
import kotlin.math.sqrt
import android.graphics.Rect as AndroidRect
import org.autojs.autojs.core.opencv.Mat as AutoJsMat
import org.autojs.autojs.runtime.api.Images as ApiImages
@@ -147,6 +152,8 @@ class Images(scriptRuntime: ScriptRuntime) : Augmentable(scriptRuntime), AsEmitt
@Suppress("MayBeConstant")
companion object {
private val TAG = Images::class.java.simpleName
@JvmField
val DEFAULT_COLOR_THRESHOLD = 4
@@ -1014,19 +1021,28 @@ class Images(scriptRuntime: ScriptRuntime) : Augmentable(scriptRuntime), AsEmitt
require(objectFeatures is ImageFeatures) {
"Argument objectFeatures ${objectFeatures.jsBrief()} for images.matchFeatures must be a ImageFeatures"
}
val matcher = opt.inquire("matcher") {
coerceIntNumber(DescriptorMatcher::class.java.getField(coerceString(it)).get(null))
} ?: DescriptorMatcher.FLANNBASED
val isObjectOrbAlike = objectFeatures.javaObject.descriptors.type() == CvType.CV_8U
val matcherType = opt.inquire("matcher") {
DescriptorMatcher::class.java.getField(coerceString(it)).get(null) as? Int
} ?: when (isObjectOrbAlike) {
/* For ORB, BRISK, AKAZE, etc. */
true -> DescriptorMatcher.BRUTEFORCE_HAMMING
/* For SIFT, SURF, etc. */
else -> DescriptorMatcher.FLANNBASED
}
val drawMatches = opt.inquire("drawMatches") { scriptRuntime.files.nonNullPath(coerceString(it)) }
val threshold = opt.inquire("threshold", ::coerceFloatNumber, 0.7f)
val threshold = opt.inquire("threshold", ::coerceFloatNumber, if (isObjectOrbAlike) 0.8f else 0.7f)
val result = ImageFeatureMatching.featureMatching(sceneFeatures.javaObject, objectFeatures.javaObject, matcher, drawMatches, threshold) ?: return@ensureArgumentsLengthInRange null
val javaMatchesImage = result.matches
val points = result.points
val result = ImageFeatureMatching.featureMatching(
/* sceneDesc = */ sceneFeatures.javaObject,
/* objectDesc = */ objectFeatures.javaObject,
/* matcherType = */ matcherType,
/* debugMatchesImagePath = */ drawMatches,
/* threshold = */ threshold,
) ?: return@ensureArgumentsLengthInRange null
if (!drawMatches.isJsNullish()) {
val matchesImage = javaMatchesImage?.let { matToImage(scriptRuntime, arrayOf(it)) }
val matchesImage = result.matches?.let { matToImage(scriptRuntime, arrayOf(it)) }
if (matchesImage != null) {
save(scriptRuntime, arrayOf(matchesImage, drawMatches, "jpg", 100))
matchesImage.recycle()
@@ -1035,17 +1051,16 @@ class Images(scriptRuntime: ScriptRuntime) : Augmentable(scriptRuntime), AsEmitt
val region = sceneFeatures.region
val scale = sceneFeatures.scale
val size = points.size
val offsetX = region.x
val offsetY = region.y
(0 until size).forEach { i ->
val point = points[i]
point.x = offsetX + point.x / scale
point.y = offsetY + point.y / scale
}
val quad = result.quad ?: return@ensureArgumentsLengthInRange null
require(quad.size == 4) { "Quad size of feature matching result must be 4 instead of ${quad.size}" }
ObjectFrame(points[0], points[1], points[3], points[2])
val (tl, tr, br, bl) = quad.map { p ->
OpencvPoint(p.x / scale + offsetX, p.y / scale + offsetY)
}
ObjectFrame(tl, tr, bl, br)
}
@JvmStatic
@@ -1439,10 +1454,7 @@ class Images(scriptRuntime: ScriptRuntime) : Augmentable(scriptRuntime), AsEmitt
// @Reference to module __images__.js from Auto.js Pro 9.3.11 by SuperMonster003 on Dec 19. 2023.
private fun fillDetectAndComputeFeaturesOptions(rows: Int, cols: Int, options: NativeObject): DetectAndComputeFeaturesOptions {
val scale = options.inquire("scale") { coerceFloatNumber(it) } ?: when {
rows * cols >= 1e6 -> 0.5f
else -> 1.0f
}
val scale = options.inquire("scale") { coerceFloatNumber(it) } ?: calcScale(rows, cols)
val cvtColor = when {
options.inquire("grayscale", ::coerceBoolean, false) -> Imgproc.COLOR_RGBA2GRAY
else -> -1
@@ -1450,7 +1462,17 @@ class Images(scriptRuntime: ScriptRuntime) : Augmentable(scriptRuntime), AsEmitt
val method = getDetectFeatureMethod(options.inquire("method", ::coerceString, "SIFT"))
val region = buildRegionInternal(options.prop("region"), cols, rows)
return DetectAndComputeFeaturesOptions(scale, cvtColor, method, region)
return DetectAndComputeFeaturesOptions(scale.coerceIn(0f, 1f), cvtColor, method, region)
}
private fun calcScale(rows: Int, cols: Int, targetArea: Int = 1_000_000, maxSide: Int = 1600): Float {
val total = rows * cols
if (total < targetArea) return 1f
val scaleByArea = sqrt(targetArea.toDouble() / total).toFloat()
val scaleBySide = maxSide.toFloat() / max(rows, cols)
return min(scaleByArea, scaleBySide).also {
Log.d(TAG, "Calculated scale: $it")
}
}
private fun extractMatPair(scriptRuntime: ScriptRuntime, argList: Array<Any?>, funcName: String): Pair<OpencvMat, OpencvMat> {

14
app/src/main/java/org/autojs/autojs/runtime/api/augment/images/ObjectFrame.kt
View File

@@ -1,5 +1,6 @@
package org.autojs.autojs.runtime.api.augment.images
import org.autojs.autojs.util.StringUtils
import org.opencv.core.Point
// @Reference to module __images__.js from Auto.js Pro 9.3.11 by SuperMonster003 on Dec 19. 2023.
@@ -20,4 +21,17 @@ class ObjectFrame(
@JvmField
val center = Point(centerX, centerY)
override fun toString(): String {
return "[${ObjectFrame::class.java.simpleName}] ${summary()}"
}
fun summary(): String = listOf(
"topLeft" to { topLeft },
"topRight" to { topRight },
"bottomLeft" to { bottomLeft },
"bottomRight" to { bottomRight },
"center" to { center },
).let { StringUtils.toFormattedSummary(it) }
}

46
app/src/main/java/org/autojs/autojs/util/ImageUtils.kt
View File

@@ -1,13 +1,17 @@
package org.autojs.autojs.util
import android.graphics.BitmapFactory
import org.opencv.core.Core
import org.opencv.core.CvType
import org.opencv.core.Mat
import org.opencv.imgproc.Imgproc
object ImageUtils {
@JvmStatic
fun calculateInSampleSize(options: BitmapFactory.Options, reqWidth: Int, reqHeight: Int): Int {
// Raw height and width of image
// Raw height and width of image (zh-CN: 图像的原始高度和宽度)
val height = options.outHeight
val width = options.outWidth
@@ -19,6 +23,7 @@ object ImageUtils {
// Calculate the largest inSampleSize value that is a power of 2 and keeps both
// height and width larger than the requested height and width.
// zh-CN: 计算满足所需高度和宽度的2的幂次方的最大采样率.
while (halfHeight / inSampleSize >= reqHeight && halfWidth / inSampleSize >= reqWidth) {
inSampleSize *= 2
}
@@ -26,4 +31,43 @@ object ImageUtils {
return inSampleSize
}
@JvmStatic
fun Mat.to8UC3(): Mat {
val src = this
// Convert bit depth to 8 bit (zh-CN: 位深转换为 8 位)
var tmp8 = Mat()
// e.g. CV_16U, CV_32F, etc (zh-CN: 例如 CV_16U, CV_32F 等)
val depth = src.depth()
if (depth != CvType.CV_8U) {
// Automatically calculate alpha/beta to map results to 0-255.
// zh-CN: 自动计算 alpha/beta, 使结果映射到 0-255.
val mm = Core.minMaxLoc(src)
val minV = mm.minVal
val maxV = mm.maxVal
// Avoid division by zero (zh-CN: 避免除以 0)
val alpha = if ((maxV - minV) < 1e-5) 1.0 else 255.0 / (maxV - minV)
val beta = -minV * alpha
// tmp8 is now 8-bit (zh-CN: 此时的 tmp8 为 8 位)
src.convertTo(tmp8, CvType.CV_8U, alpha, beta)
} else {
// Already 8-bit, continue using (zh-CN: 已经是 8 位, 继续使用)
tmp8 = src
}
// Convert channel count to 3 (zh-CN: 通道数转换为 3)
val ch = tmp8.channels()
var dst = Mat()
when (ch) {
1 -> Imgproc.cvtColor(tmp8, dst, Imgproc.COLOR_GRAY2BGR)
3 -> dst = tmp8.clone()
4 -> Imgproc.cvtColor(tmp8, dst, Imgproc.COLOR_BGRA2BGR)
else -> throw IllegalArgumentException("Unsupported channel count: $ch")
}
return dst
}
}

26
app/src/main/java/org/autojs/autojs/util/StringUtils.kt
View File

@@ -5,6 +5,8 @@ import android.text.TextUtils
import org.autojs.autojs.annotation.LocaleNonRelated
import org.autojs.autojs.app.GlobalAppContext
import org.autojs.autojs.core.pref.Language
import org.autojs.autojs.extension.NumberExtensions.roundToString
import org.opencv.core.Point
import java.util.Locale
import kotlin.math.min
import kotlin.math.pow
@@ -204,4 +206,28 @@ object StringUtils {
@Deprecated("Deprecated since v6.6.0", ReplaceWith("uppercaseFirstChar(s)"))
fun toUpperCaseFirst(s: String) = uppercaseFirstChar(s)
@JvmStatic
fun toFormattedSummary(dataList: List<Pair<String, () -> Any?>>): String {
val separatorLv0 = "\n"
val separatorLv1 = "$separatorLv0 "
val separatorLv2 = "$separatorLv1 "
return dataList.joinToString(prefix = "{$separatorLv1", separator = separatorLv1, postfix = "$separatorLv0}") { (name, action) ->
val value = when (val actionResult = action()) {
is CharSequence -> "\"$actionResult\""
is Iterable<*> -> actionResult.joinToString(prefix = "[$separatorLv2", separator = separatorLv2, postfix = "$separatorLv1]")
is Array<*> -> actionResult.joinToString(prefix = "[$separatorLv2", separator = separatorLv2, postfix = "$separatorLv1]")
is Point -> actionResult.toFormattedPointString(0)
else -> actionResult
}
"$name=$value"
}
}
@JvmStatic
@JvmOverloads
fun Point.toFormattedPointString(scale: Int = 0): String {
return "{${x.roundToString(scale)}, ${y.roundToString(scale)}}"
}
}