tt/VscoolOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

version:2.1.7

Browse Source 
bugfixes:
update:优化ocr
This commit is contained in:
tongtongstudio
2025-06-03 14:41:01 +08:00
parent d7e830985f
commit 3697aff8dc
2 changed files with 262 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/build.gradle
View File

@@ -18,8 +18,8 @@ android {
minSdkVersion 24 minSdkVersion 24
targetSdkVersion 29 targetSdkVersion 29
versionCode 216 versionCode 217
versionName "2.1.6" versionName "2.1.7"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"

336
app/src/main/java/com/vscool/os/service/WeAccessibilityService.java
View File

@@ -24,10 +24,11 @@ import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager; import android.media.projection.MediaProjectionManager;
import android.os.Build; import android.os.Build;
import android.os.Handler; import android.os.Handler;
import android.os.Looper;
import android.text.TextUtils; import android.text.TextUtils;
import android.util.DisplayMetrics; import android.util.DisplayMetrics;
import android.util.Log; import android.util.Log;
import android.view.Surface;
import android.view.SurfaceControl;
import android.view.WindowManager; import android.view.WindowManager;
import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo;
@@ -125,6 +126,7 @@ public class WeAccessibilityService extends AccessibilityService {
private String mTagName = "";//微信联系人标签名 private String mTagName = "";//微信联系人标签名
private boolean mAutoAccept = false; private boolean mAutoAccept = false;
private boolean finished = true; private boolean finished = true;
private boolean answered = false;
private Handler handler = null; private Handler handler = null;
private AccessibilityEvent input = null; private AccessibilityEvent input = null;
@@ -199,6 +201,7 @@ public class WeAccessibilityService extends AccessibilityService {
} else { } else {
Log.e(TAG, "onCreate: load model failed"); Log.e(TAG, "onCreate: load model failed");
} }
initAutoAcceptObservable();
} }
@Override @Override
@@ -342,6 +345,12 @@ public class WeAccessibilityService extends AccessibilityService {
void onImage(Image image); void onImage(Image image);
} }
private processBitmap mProcessBitmap;
public interface processBitmap {
void onImage(Bitmap bitmap);
}
private boolean completed = true; private boolean completed = true;
private void convertImage() { private void convertImage() {
@@ -353,35 +362,27 @@ public class WeAccessibilityService extends AccessibilityService {
public void onImage(Image image) { public void onImage(Image image) {
Log.e(TAG, "onImage: time = " + System.currentTimeMillis()); Log.e(TAG, "onImage: time = " + System.currentTimeMillis());
// 获取最新的图像 // 获取最新的图像
if (image != null) { Log.e(TAG, "startScreenCapture: image width = " + image.getWidth());
Log.e(TAG, "startScreenCapture: image width = " + image.getWidth()); Log.e(TAG, "startScreenCapture: image height = " + image.getHeight());
Log.e(TAG, "startScreenCapture: image height = " + image.getHeight()); // Image.Plane[] planes = image.getPlanes();
Image.Plane[] planes = image.getPlanes(); // 将 Image 转换为 Bitmap
if (planes.length > 0) { Bitmap bitmap = convertImageToBitmap(image);
// 将 Image 转换为 Bitmap // 释放资源
Bitmap bitmap = convertImageToBitmap(image); image.close();
// 处理获取到的屏幕图像
if (bitmap != null) { // 处理获取到的屏幕图像
Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth()); Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth());
Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight()); Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight());
predictor.setInputImage(bitmap); predictor.setInputImage(bitmap);
if (onRunModel()) { if (onRunModel()) {
Bitmap outputImage = predictor.outputImage(); Bitmap outputImage = predictor.outputImage();
Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth()); Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth());
Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight()); Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight());
ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels(); ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels();
emitter.onNext(ocrResultModels); emitter.onNext(ocrResultModels);
Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms"); Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms");
} else {
Log.e(TAG, "convertImage: failed");
completed = true;
}
}
}
// 释放资源
image.close();
} else { } else {
Log.e(TAG, "onImage: image is null"); Log.e(TAG, "convertImage: failed");
completed = true; completed = true;
} }
} }
@@ -399,10 +400,9 @@ public class WeAccessibilityService extends AccessibilityService {
@Override @Override
public void onNext(ArrayList<OcrResultModel> ocrResultModels) { public void onNext(ArrayList<OcrResultModel> ocrResultModels) {
Log.e(TAG, "onNext: ocrResultList size = " + ocrResultModels.size()); Log.e("convertImage", "onNext: ocrResultList size = " + ocrResultModels.size());
Log.v(TAG, "onNext: " + ocrResultModels); Log.v("convertImage", "onNext: " + ocrResultModels);
analysisUi(ocrResultModels); analysisUi(ocrResultModels);
completed = true;
} }
@Override @Override
@@ -419,7 +419,6 @@ public class WeAccessibilityService extends AccessibilityService {
}); });
} }
private void analysisUi(ArrayList<OcrResultModel> ocrResultModels) { private void analysisUi(ArrayList<OcrResultModel> ocrResultModels) {
String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this); String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this);
Log.e(TAG, "analysisUi: topAppClassName = " + topAppClassName); Log.e(TAG, "analysisUi: topAppClassName = " + topAppClassName);
@@ -462,25 +461,39 @@ public class WeAccessibilityService extends AccessibilityService {
if (stringList.contains("通讯录")) {//在通讯录页面 if (stringList.contains("通讯录")) {//在通讯录页面
if (contactSize >= 2) { if (contactSize >= 2) {
equalsViewTouch(ocrResultModels, topAppClassName, "标签"); equalsViewTouch(ocrResultModels, topAppClassName, "标签");
} else {
completed = true;
} }
} else {
completed = true;
} }
} }
break; break;
case "com.tencent.mm.plugin.label.ui.ContactLabelManagerUI"://标签页面 case "com.tencent.mm.plugin.label.ui.ContactLabelManagerUI"://标签页面
if (stringList.contains("通讯录标签")) {//在标签里面 if (stringList.contains("通讯录标签")) {//在标签里面
containsViewTouch(ocrResultModels, topAppClassName, mTagName); boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, mTagName);
if (!clicked) {
completed = true; // 点击失败时恢复处理
}
} else {
completed = true;
} }
break; break;
case "com.tencent.mm.ui.mvvm.MvvmContactListUI"://标签联系人列表 case "com.tencent.mm.ui.mvvm.MvvmContactListUI"://标签联系人列表
if (stringList.contains(mName) && findString(stringList, mTagName)) { if (stringList.contains(mName) && findString(stringList, mTagName)) {
equalsViewTouch(ocrResultModels, topAppClassName, mName); boolean clicked = equalsViewTouch(ocrResultModels, topAppClassName, mName);
if (!clicked) {
completed = true; // 点击失败时恢复处理
}
} else { } else {
//滑动啥的 //滑动啥的
scrollDown(); // scrollDown();
completed = true; // 滚动后立即允许下一帧处理
} }
break; break;
case "com.tencent.mm.plugin.profile.ui.ContactInfoUI"://个人信息页面 case "com.tencent.mm.plugin.profile.ui.ContactInfoUI"://个人信息页面
case "com.tencent.mm.ui.widget.dialog.w3":
if (findString(stringList, mName) && findString(stringList, DIALER_TEXT)) { if (findString(stringList, mName) && findString(stringList, DIALER_TEXT)) {
if (findString(stringList, CALL_TEXT) && findString(stringList, VIDEO_TEXT)) { if (findString(stringList, CALL_TEXT) && findString(stringList, VIDEO_TEXT)) {
String callName; String callName;
@@ -489,15 +502,13 @@ public class WeAccessibilityService extends AccessibilityService {
} else { } else {
callName = CALL_TEXT; callName = CALL_TEXT;
} }
boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName); boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: successful = " + successful); Log.e(TAG, "analysisUi: clicked = " + clicked);
if (!clicked) {
completed = true;
}
} else { } else {
new Handler(Looper.getMainLooper()).postDelayed(new Runnable() { containsViewTouchDelayed(ocrResultModels, topAppClassName, DIALER_TEXT, 1234);
@Override
public void run() {
containsViewTouch(ocrResultModels, topAppClassName, DIALER_TEXT);
}
}, 2000);
} }
} else if (findString(stringList, CALL_TEXT) || findString(stringList, VIDEO_TEXT)) { } else if (findString(stringList, CALL_TEXT) || findString(stringList, VIDEO_TEXT)) {
String callName; String callName;
@@ -506,12 +517,13 @@ public class WeAccessibilityService extends AccessibilityService {
} else { } else {
callName = CALL_TEXT; callName = CALL_TEXT;
} }
boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName); boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: successful = " + successful); Log.e(TAG, "analysisUi: clicked = " + clicked);
// if (successful) {//有可能太快了点击不生效 if (!clicked) {
// Toaster.showLong("拨打完成"); completed = true;
// releaseDisplay(); }
// } } else {
completed = true;
} }
break; break;
@@ -540,6 +552,7 @@ public class WeAccessibilityService extends AccessibilityService {
releaseDisplay(); releaseDisplay();
break; break;
default: default:
completed = true; // 确保未知界面不阻塞流程
} }
// if (stringList.contains("微信")) {//在主页 // if (stringList.contains("微信")) {//在主页
@@ -591,6 +604,112 @@ public class WeAccessibilityService extends AccessibilityService {
// } // }
} }
private void initAutoAcceptObservable() {
Observable.create(new ObservableOnSubscribe<ArrayList<OcrResultModel>>() {
@Override
public void subscribe(@NonNull ObservableEmitter<ArrayList<OcrResultModel>> emitter) throws Throwable {
mProcessBitmap = new processBitmap() {
@Override
public void onImage(Bitmap bitmap) {
Log.e(TAG, "initAutoAcceptObservable onImage: time = " + System.currentTimeMillis());
Log.e(TAG, "initAutoAcceptObservable: bitmap width = " + bitmap.getWidth());
Log.e(TAG, "initAutoAcceptObservable: bitmap height = " + bitmap.getHeight());
predictor.setInputImage(bitmap);
if (onRunModel()) {
Bitmap outputImage = predictor.outputImage();
Log.e(TAG, "initAutoAcceptObservable: outputImage width = " + outputImage.getWidth());
Log.e(TAG, "initAutoAcceptObservable: outputImage height = " + outputImage.getHeight());
ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels();
emitter.onNext(ocrResultModels);
Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms");
} else {
Log.e(TAG, "convertImage: failed");
completed = true;
}
}
};
}
}).subscribeOn(Schedulers.computation())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(new Observer<ArrayList<OcrResultModel>>() {
@Override
public void onSubscribe(Disposable d) {
Log.e("convertImage bitmap", "onSubscribe: ");
}
@Override
public void onNext(ArrayList<OcrResultModel> ocrResultModels) {
Log.e("convertImage bitmap", "onNext: ocrResultList size = " + ocrResultModels.size());
Log.v("convertImage bitmap", "onNext: " + ocrResultModels);
analysisAutoAccept(ocrResultModels);
}
@Override
public void onError(Throwable e) {
Log.e("convertImage bitmap", "onError: " + e.getMessage());
completed = true;
}
@Override
public void onComplete() {
Log.e("convertImage bitmap", "onComplete: ");
completed = true;
}
});
}
private void analysisAutoAccept(ArrayList<OcrResultModel> ocrResultModels) {
String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this);
Log.e(TAG, "analysisAutoAccept: topAppClassName = " + topAppClassName);
Log.e(TAG, "analysisAutoAccept: mCurrentClassName = " + mCurrentClassName);
Log.e(TAG, "analysisAutoAccept: mCurrentPackageName = " + mCurrentPackageName);
if (!"com.tencent.mm".equals(mCurrentPackageName)) {
completed = true; // 非微信应用，直接允许处理下一张
return;
}
if (ocrResultModels == null || ocrResultModels.isEmpty()) {
Log.e(TAG, "analysisAutoAccept: ocrResultModels empty");
completed = true;
return;
}
List<String> stringList = ocrResultModels.stream().map(new Function<OcrResultModel, String>() {
@Override
public String apply(OcrResultModel ocrResultModel) {
return ocrResultModel.getLabel();
}
}).collect(Collectors.toList());
switch (topAppClassName) {
case "com.tencent.mm.plugin.voip.ui.VideoActivity":
if (stringList.contains("接听")) {//在标签里面
Optional<OcrResultModel> ocrResultOptional = ocrResultModels.stream().filter(new Predicate<OcrResultModel>() {
@Override
public boolean test(OcrResultModel ocrResultModel) {
return "接听".equals(ocrResultModel.getLabel());
}
}).findAny();
if (ocrResultOptional.isPresent()) {
OcrResultModel ocrResult = ocrResultOptional.get();
List<Point> points = ocrResult.getPoints();
Log.e(TAG, "analysisAutoAccept: points = " + points);
Point point = getCenterPoint(points);
Log.e(TAG, "analysisAutoAccept: CenterPoint = " + point);
clickByPoint(point.x, point.y - 75, topAppClassName, new Runnable() {
@Override
public void run() {
answered = true;
mCurrentStep = Step.WAITING;
Toast.makeText(WeAccessibilityService.this, "已自动接听视频/语音", Toast.LENGTH_LONG).show();
}
});
}
}
break;
default:
}
}
private boolean clickByPoint(int x, int y, String className, Runnable onComplete) { private boolean clickByPoint(int x, int y, String className, Runnable onComplete) {
Log.e(TAG, "clickByNode: x = " + x); Log.e(TAG, "clickByNode: x = " + x);
Log.e(TAG, "clickByNode: y = " + y); Log.e(TAG, "clickByNode: y = " + y);
@@ -668,6 +787,8 @@ public class WeAccessibilityService extends AccessibilityService {
completed = true; completed = true;
} }
}); });
} else {
completed = true;
} }
return false; return false;
} }
@@ -691,6 +812,38 @@ public class WeAccessibilityService extends AccessibilityService {
completed = true; completed = true;
} }
}); });
} else {
completed = true;
}
return false;
}
private boolean containsViewTouchDelayed(ArrayList<OcrResultModel> ocrResultModels, String className, String text, long millisecond) {
Optional<OcrResultModel> ocrResultOptional = ocrResultModels.stream().filter(new Predicate<OcrResultModel>() {
@Override
public boolean test(OcrResultModel ocrResultModel) {
return ocrResultModel.getLabel().contains(text);
}
}).findAny();
if (ocrResultOptional.isPresent()) {
OcrResultModel ocrResult = ocrResultOptional.get();
List<Point> points = ocrResult.getPoints();
Log.e(TAG, "analysisUi: points = " + points);
Point point = getCenterPoint(points);
Log.e(TAG, "analysisUi: CenterPoint = " + point);
return clickByPoint(point.x, point.y, className, new Runnable() {
@Override
public void run() {
new Handler().postDelayed(new Runnable() {
@Override
public void run() {
completed = true;
}
}, millisecond);
}
});
} else {
completed = true;
} }
return false; return false;
} }
@@ -769,6 +922,32 @@ public class WeAccessibilityService extends AccessibilityService {
return predictor.isLoaded() && predictor.runModel(1, 0, 1); return predictor.isLoaded() && predictor.runModel(1, 0, 1);
} }
// 获取屏幕截图（包含状态栏和导航栏）
public void takeScreenshot() {
try {
// 获取DisplayMetrics
DisplayMetrics metrics = new DisplayMetrics();
WindowManager wm = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
wm.getDefaultDisplay().getRealMetrics(metrics);
// 反射调用SurfaceControl.screenshot()
// Class<?> surfaceControl = Class.forName("android.view.SurfaceControl");
// Method screenshotMethod = surfaceControl.getMethod("screenshot", int.class, int.class);
// Bitmap bitmap = (Bitmap) screenshotMethod.invoke(null, metrics.widthPixels, metrics.heightPixels);
// 处理旋转状态（如横屏）
// int rotation = wm.getDefaultDisplay().getRotation();
// if (rotation != Surface.ROTATION_0) {
// Matrix matrix = new Matrix();
// matrix.postRotate(90 * rotation);
// bitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
// }
Bitmap bitmap = SurfaceControl.screenshot(new Rect(), metrics.widthPixels, metrics.heightPixels, Surface.ROTATION_0);
mProcessBitmap.onImage(bitmap);
} catch (Exception e) {
e.printStackTrace();
}
}
/** /**
* 1.在微信页面直接找到联系人拨打电话 * 1.在微信页面直接找到联系人拨打电话
* 2.在联系人页面找到并拨打 * 2.在联系人页面找到并拨打
@@ -805,31 +984,34 @@ public class WeAccessibilityService extends AccessibilityService {
} }
} }
} }
// switch (mCurrentStep) { switch (mCurrentStep) {
// case WAITING: case WAITING:
// mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false); mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false);
// Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept); Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept);
// if (!mAutoAccept) { if (!mAutoAccept) {
// return; return;
// } }
// if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) { if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) {
// mCurrentStep = Step.WECHAT_HANDS_FREE; mCurrentStep = Step.WECHAT_HANDS_FREE;
// Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show(); Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show();
// } else { answered = true;
// mCurrentStep = Step.WAITING; } else {
//// clickAnswer(); mCurrentStep = Step.WAITING;
// } // clickAnswer();
// break; // if (!answered)
// case WECHAT_HANDS_FREE: takeScreenshot();
// handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT); }
// break; break;
// case DIALER_HANDS_FREE: case WECHAT_HANDS_FREE:
// if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) { handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT);
// dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT); break;
// } else { case DIALER_HANDS_FREE:
// mCurrentStep = Step.WAITING; if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) {
// } dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT);
// break; } else {
mCurrentStep = Step.WAITING;
}
break;
// case CLICK_HOME://主页能找到直接点击进去更多 // case CLICK_HOME://主页能找到直接点击进去更多
// stepHome(Property.TEXT, mName); // stepHome(Property.TEXT, mName);
// break; // break;
@@ -876,7 +1058,7 @@ public class WeAccessibilityService extends AccessibilityService {
//// } //// }
//// break; //// break;
// default: // default:
// } }
} }
@Override @Override
@@ -1407,6 +1589,8 @@ public class WeAccessibilityService extends AccessibilityService {
if (STOP_RECORD_ACTION.equals(intent.getAction())) { if (STOP_RECORD_ACTION.equals(intent.getAction())) {
mCurrentPackageName = ""; mCurrentPackageName = "";
mCurrentClassName = ""; mCurrentClassName = "";
completed = true;
answered = false;
releaseDisplay(); releaseDisplay();
} }
} }