tt/VscoolOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

version:2.1.7

Browse Source 
bugfixes:
update:优化ocr
This commit is contained in:
tongtongstudio
2025-06-03 14:41:01 +08:00
parent d7e830985f
commit 3697aff8dc
2 changed files with 262 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/build.gradle
View File

@@ -18,8 +18,8 @@ android {
minSdkVersion 24
targetSdkVersion 29
versionCode 216
versionName "2.1.6"
versionCode 217
versionName "2.1.7"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"

336
app/src/main/java/com/vscool/os/service/WeAccessibilityService.java
View File

@@ -24,10 +24,11 @@ import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.text.TextUtils;
import android.util.DisplayMetrics;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceControl;
import android.view.WindowManager;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;
@@ -125,6 +126,7 @@ public class WeAccessibilityService extends AccessibilityService {
private String mTagName = "";//微信联系人标签名
private boolean mAutoAccept = false;
private boolean finished = true;
private boolean answered = false;
private Handler handler = null;
private AccessibilityEvent input = null;
@@ -199,6 +201,7 @@ public class WeAccessibilityService extends AccessibilityService {
} else {
Log.e(TAG, "onCreate: load model failed");
}
initAutoAcceptObservable();
}
@Override
@@ -342,6 +345,12 @@ public class WeAccessibilityService extends AccessibilityService {
void onImage(Image image);
}
private processBitmap mProcessBitmap;
public interface processBitmap {
void onImage(Bitmap bitmap);
}
private boolean completed = true;
private void convertImage() {
@@ -353,35 +362,27 @@ public class WeAccessibilityService extends AccessibilityService {
public void onImage(Image image) {
Log.e(TAG, "onImage: time = " + System.currentTimeMillis());
// 获取最新的图像
if (image != null) {
Log.e(TAG, "startScreenCapture: image width = " + image.getWidth());
Log.e(TAG, "startScreenCapture: image height = " + image.getHeight());
Image.Plane[] planes = image.getPlanes();
if (planes.length > 0) {
// 将 Image 转换为 Bitmap
Bitmap bitmap = convertImageToBitmap(image);
// 处理获取到的屏幕图像
if (bitmap != null) {
Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth());
Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight());
predictor.setInputImage(bitmap);
if (onRunModel()) {
Bitmap outputImage = predictor.outputImage();
Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth());
Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight());
ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels();
emitter.onNext(ocrResultModels);
Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms");
} else {
Log.e(TAG, "convertImage: failed");
completed = true;
}
}
}
// 释放资源
image.close();
Log.e(TAG, "startScreenCapture: image width = " + image.getWidth());
Log.e(TAG, "startScreenCapture: image height = " + image.getHeight());
// Image.Plane[] planes = image.getPlanes();
// 将 Image 转换为 Bitmap
Bitmap bitmap = convertImageToBitmap(image);
// 释放资源
image.close();
// 处理获取到的屏幕图像
Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth());
Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight());
predictor.setInputImage(bitmap);
if (onRunModel()) {
Bitmap outputImage = predictor.outputImage();
Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth());
Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight());
ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels();
emitter.onNext(ocrResultModels);
Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms");
} else {
Log.e(TAG, "onImage: image is null");
Log.e(TAG, "convertImage: failed");
completed = true;
}
}
@@ -399,10 +400,9 @@ public class WeAccessibilityService extends AccessibilityService {
@Override
public void onNext(ArrayList<OcrResultModel> ocrResultModels) {
Log.e(TAG, "onNext: ocrResultList size = " + ocrResultModels.size());
Log.v(TAG, "onNext: " + ocrResultModels);
Log.e("convertImage", "onNext: ocrResultList size = " + ocrResultModels.size());
Log.v("convertImage", "onNext: " + ocrResultModels);
analysisUi(ocrResultModels);
completed = true;
}
@Override
@@ -419,7 +419,6 @@ public class WeAccessibilityService extends AccessibilityService {
});
}
private void analysisUi(ArrayList<OcrResultModel> ocrResultModels) {
String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this);
Log.e(TAG, "analysisUi: topAppClassName = " + topAppClassName);
@@ -462,25 +461,39 @@ public class WeAccessibilityService extends AccessibilityService {
if (stringList.contains("通讯录")) {//在通讯录页面
if (contactSize >= 2) {
equalsViewTouch(ocrResultModels, topAppClassName, "标签");
} else {
completed = true;
}
} else {
completed = true;
}
}
break;
case "com.tencent.mm.plugin.label.ui.ContactLabelManagerUI"://标签页面
if (stringList.contains("通讯录标签")) {//在标签里面
containsViewTouch(ocrResultModels, topAppClassName, mTagName);
boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, mTagName);
if (!clicked) {
completed = true; // 点击失败时恢复处理
}
} else {
completed = true;
}
break;
case "com.tencent.mm.ui.mvvm.MvvmContactListUI"://标签联系人列表
if (stringList.contains(mName) && findString(stringList, mTagName)) {
equalsViewTouch(ocrResultModels, topAppClassName, mName);
boolean clicked = equalsViewTouch(ocrResultModels, topAppClassName, mName);
if (!clicked) {
completed = true; // 点击失败时恢复处理
}
} else {
//滑动啥的
scrollDown();
// scrollDown();
completed = true; // 滚动后立即允许下一帧处理
}
break;
case "com.tencent.mm.plugin.profile.ui.ContactInfoUI"://个人信息页面
case "com.tencent.mm.ui.widget.dialog.w3":
if (findString(stringList, mName) && findString(stringList, DIALER_TEXT)) {
if (findString(stringList, CALL_TEXT) && findString(stringList, VIDEO_TEXT)) {
String callName;
@@ -489,15 +502,13 @@ public class WeAccessibilityService extends AccessibilityService {
} else {
callName = CALL_TEXT;
}
boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: successful = " + successful);
boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: clicked = " + clicked);
if (!clicked) {
completed = true;
}
} else {
new Handler(Looper.getMainLooper()).postDelayed(new Runnable() {
@Override
public void run() {
containsViewTouch(ocrResultModels, topAppClassName, DIALER_TEXT);
}
}, 2000);
containsViewTouchDelayed(ocrResultModels, topAppClassName, DIALER_TEXT, 1234);
}
} else if (findString(stringList, CALL_TEXT) || findString(stringList, VIDEO_TEXT)) {
String callName;
@@ -506,12 +517,13 @@ public class WeAccessibilityService extends AccessibilityService {
} else {
callName = CALL_TEXT;
}
boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: successful = " + successful);
// if (successful) {//有可能太快了点击不生效
// Toaster.showLong("拨打完成");
// releaseDisplay();
// }
boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName);
Log.e(TAG, "analysisUi: clicked = " + clicked);
if (!clicked) {
completed = true;
}
} else {
completed = true;
}
break;
@@ -540,6 +552,7 @@ public class WeAccessibilityService extends AccessibilityService {
releaseDisplay();
break;
default:
completed = true; // 确保未知界面不阻塞流程
}
// if (stringList.contains("微信")) {//在主页
@@ -591,6 +604,112 @@ public class WeAccessibilityService extends AccessibilityService {
// }
}
private void initAutoAcceptObservable() {
Observable.create(new ObservableOnSubscribe<ArrayList<OcrResultModel>>() {
@Override
public void subscribe(@NonNull ObservableEmitter<ArrayList<OcrResultModel>> emitter) throws Throwable {
mProcessBitmap = new processBitmap() {
@Override
public void onImage(Bitmap bitmap) {
Log.e(TAG, "initAutoAcceptObservable onImage: time = " + System.currentTimeMillis());
Log.e(TAG, "initAutoAcceptObservable: bitmap width = " + bitmap.getWidth());
Log.e(TAG, "initAutoAcceptObservable: bitmap height = " + bitmap.getHeight());
predictor.setInputImage(bitmap);
if (onRunModel()) {
Bitmap outputImage = predictor.outputImage();
Log.e(TAG, "initAutoAcceptObservable: outputImage width = " + outputImage.getWidth());
Log.e(TAG, "initAutoAcceptObservable: outputImage height = " + outputImage.getHeight());
ArrayList<OcrResultModel> ocrResultModels = predictor.getOcrResultModels();
emitter.onNext(ocrResultModels);
Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms");
} else {
Log.e(TAG, "convertImage: failed");
completed = true;
}
}
};
}
}).subscribeOn(Schedulers.computation())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(new Observer<ArrayList<OcrResultModel>>() {
@Override
public void onSubscribe(Disposable d) {
Log.e("convertImage bitmap", "onSubscribe: ");
}
@Override
public void onNext(ArrayList<OcrResultModel> ocrResultModels) {
Log.e("convertImage bitmap", "onNext: ocrResultList size = " + ocrResultModels.size());
Log.v("convertImage bitmap", "onNext: " + ocrResultModels);
analysisAutoAccept(ocrResultModels);
}
@Override
public void onError(Throwable e) {
Log.e("convertImage bitmap", "onError: " + e.getMessage());
completed = true;
}
@Override
public void onComplete() {
Log.e("convertImage bitmap", "onComplete: ");
completed = true;
}
});
}
private void analysisAutoAccept(ArrayList<OcrResultModel> ocrResultModels) {
String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this);
Log.e(TAG, "analysisAutoAccept: topAppClassName = " + topAppClassName);
Log.e(TAG, "analysisAutoAccept: mCurrentClassName = " + mCurrentClassName);
Log.e(TAG, "analysisAutoAccept: mCurrentPackageName = " + mCurrentPackageName);
if (!"com.tencent.mm".equals(mCurrentPackageName)) {
completed = true; // 非微信应用，直接允许处理下一张
return;
}
if (ocrResultModels == null || ocrResultModels.isEmpty()) {
Log.e(TAG, "analysisAutoAccept: ocrResultModels empty");
completed = true;
return;
}
List<String> stringList = ocrResultModels.stream().map(new Function<OcrResultModel, String>() {
@Override
public String apply(OcrResultModel ocrResultModel) {
return ocrResultModel.getLabel();
}
}).collect(Collectors.toList());
switch (topAppClassName) {
case "com.tencent.mm.plugin.voip.ui.VideoActivity":
if (stringList.contains("接听")) {//在标签里面
Optional<OcrResultModel> ocrResultOptional = ocrResultModels.stream().filter(new Predicate<OcrResultModel>() {
@Override
public boolean test(OcrResultModel ocrResultModel) {
return "接听".equals(ocrResultModel.getLabel());
}
}).findAny();
if (ocrResultOptional.isPresent()) {
OcrResultModel ocrResult = ocrResultOptional.get();
List<Point> points = ocrResult.getPoints();
Log.e(TAG, "analysisAutoAccept: points = " + points);
Point point = getCenterPoint(points);
Log.e(TAG, "analysisAutoAccept: CenterPoint = " + point);
clickByPoint(point.x, point.y - 75, topAppClassName, new Runnable() {
@Override
public void run() {
answered = true;
mCurrentStep = Step.WAITING;
Toast.makeText(WeAccessibilityService.this, "已自动接听视频/语音", Toast.LENGTH_LONG).show();
}
});
}
}
break;
default:
}
}
private boolean clickByPoint(int x, int y, String className, Runnable onComplete) {
Log.e(TAG, "clickByNode: x = " + x);
Log.e(TAG, "clickByNode: y = " + y);
@@ -668,6 +787,8 @@ public class WeAccessibilityService extends AccessibilityService {
completed = true;
}
});
} else {
completed = true;
}
return false;
}
@@ -691,6 +812,38 @@ public class WeAccessibilityService extends AccessibilityService {
completed = true;
}
});
} else {
completed = true;
}
return false;
}
private boolean containsViewTouchDelayed(ArrayList<OcrResultModel> ocrResultModels, String className, String text, long millisecond) {
Optional<OcrResultModel> ocrResultOptional = ocrResultModels.stream().filter(new Predicate<OcrResultModel>() {
@Override
public boolean test(OcrResultModel ocrResultModel) {
return ocrResultModel.getLabel().contains(text);
}
}).findAny();
if (ocrResultOptional.isPresent()) {
OcrResultModel ocrResult = ocrResultOptional.get();
List<Point> points = ocrResult.getPoints();
Log.e(TAG, "analysisUi: points = " + points);
Point point = getCenterPoint(points);
Log.e(TAG, "analysisUi: CenterPoint = " + point);
return clickByPoint(point.x, point.y, className, new Runnable() {
@Override
public void run() {
new Handler().postDelayed(new Runnable() {
@Override
public void run() {
completed = true;
}
}, millisecond);
}
});
} else {
completed = true;
}
return false;
}
@@ -769,6 +922,32 @@ public class WeAccessibilityService extends AccessibilityService {
return predictor.isLoaded() && predictor.runModel(1, 0, 1);
}
// 获取屏幕截图（包含状态栏和导航栏）
public void takeScreenshot() {
try {
// 获取DisplayMetrics
DisplayMetrics metrics = new DisplayMetrics();
WindowManager wm = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
wm.getDefaultDisplay().getRealMetrics(metrics);
// 反射调用SurfaceControl.screenshot()
// Class<?> surfaceControl = Class.forName("android.view.SurfaceControl");
// Method screenshotMethod = surfaceControl.getMethod("screenshot", int.class, int.class);
// Bitmap bitmap = (Bitmap) screenshotMethod.invoke(null, metrics.widthPixels, metrics.heightPixels);
// 处理旋转状态（如横屏）
// int rotation = wm.getDefaultDisplay().getRotation();
// if (rotation != Surface.ROTATION_0) {
// Matrix matrix = new Matrix();
// matrix.postRotate(90 * rotation);
// bitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
// }
Bitmap bitmap = SurfaceControl.screenshot(new Rect(), metrics.widthPixels, metrics.heightPixels, Surface.ROTATION_0);
mProcessBitmap.onImage(bitmap);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 1.在微信页面直接找到联系人拨打电话
* 2.在联系人页面找到并拨打
@@ -805,31 +984,34 @@ public class WeAccessibilityService extends AccessibilityService {
}
}
}
// switch (mCurrentStep) {
// case WAITING:
// mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false);
// Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept);
// if (!mAutoAccept) {
// return;
// }
// if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) {
// mCurrentStep = Step.WECHAT_HANDS_FREE;
// Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show();
// } else {
// mCurrentStep = Step.WAITING;
//// clickAnswer();
// }
// break;
// case WECHAT_HANDS_FREE:
// handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT);
// break;
// case DIALER_HANDS_FREE:
// if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) {
// dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT);
// } else {
// mCurrentStep = Step.WAITING;
// }
// break;
switch (mCurrentStep) {
case WAITING:
mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false);
Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept);
if (!mAutoAccept) {
return;
}
if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) {
mCurrentStep = Step.WECHAT_HANDS_FREE;
Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show();
answered = true;
} else {
mCurrentStep = Step.WAITING;
// clickAnswer();
// if (!answered)
takeScreenshot();
}
break;
case WECHAT_HANDS_FREE:
handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT);
break;
case DIALER_HANDS_FREE:
if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) {
dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT);
} else {
mCurrentStep = Step.WAITING;
}
break;
// case CLICK_HOME://主页能找到直接点击进去更多
// stepHome(Property.TEXT, mName);
// break;
@@ -876,7 +1058,7 @@ public class WeAccessibilityService extends AccessibilityService {
//// }
//// break;
// default:
// }
}
}
@Override
@@ -1407,6 +1589,8 @@ public class WeAccessibilityService extends AccessibilityService {
if (STOP_RECORD_ACTION.equals(intent.getAction())) {
mCurrentPackageName = "";
mCurrentClassName = "";
completed = true;
answered = false;
releaseDisplay();
}
}