diff --git a/app/build.gradle b/app/build.gradle index d01701f..bf8d14b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -18,8 +18,8 @@ android { minSdkVersion 24 targetSdkVersion 29 - versionCode 216 - versionName "2.1.6" + versionCode 217 + versionName "2.1.7" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" diff --git a/app/src/main/java/com/vscool/os/service/WeAccessibilityService.java b/app/src/main/java/com/vscool/os/service/WeAccessibilityService.java index 7ebb598..9ba7e44 100644 --- a/app/src/main/java/com/vscool/os/service/WeAccessibilityService.java +++ b/app/src/main/java/com/vscool/os/service/WeAccessibilityService.java @@ -24,10 +24,11 @@ import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; import android.os.Build; import android.os.Handler; -import android.os.Looper; import android.text.TextUtils; import android.util.DisplayMetrics; import android.util.Log; +import android.view.Surface; +import android.view.SurfaceControl; import android.view.WindowManager; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityNodeInfo; @@ -125,6 +126,7 @@ public class WeAccessibilityService extends AccessibilityService { private String mTagName = "";//微信联系人标签名 private boolean mAutoAccept = false; private boolean finished = true; + private boolean answered = false; private Handler handler = null; private AccessibilityEvent input = null; @@ -199,6 +201,7 @@ public class WeAccessibilityService extends AccessibilityService { } else { Log.e(TAG, "onCreate: load model failed"); } + initAutoAcceptObservable(); } @Override @@ -342,6 +345,12 @@ public class WeAccessibilityService extends AccessibilityService { void onImage(Image image); } + private processBitmap mProcessBitmap; + + public interface processBitmap { + void onImage(Bitmap bitmap); + } + private boolean completed = true; private void convertImage() { @@ -353,35 +362,27 @@ public class WeAccessibilityService extends AccessibilityService { public void onImage(Image image) { Log.e(TAG, "onImage: time = " + System.currentTimeMillis()); // 获取最新的图像 - if (image != null) { - Log.e(TAG, "startScreenCapture: image width = " + image.getWidth()); - Log.e(TAG, "startScreenCapture: image height = " + image.getHeight()); - Image.Plane[] planes = image.getPlanes(); - if (planes.length > 0) { - // 将 Image 转换为 Bitmap - Bitmap bitmap = convertImageToBitmap(image); - // 处理获取到的屏幕图像 - if (bitmap != null) { - Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth()); - Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight()); - predictor.setInputImage(bitmap); - if (onRunModel()) { - Bitmap outputImage = predictor.outputImage(); - Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth()); - Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight()); - ArrayList ocrResultModels = predictor.getOcrResultModels(); - emitter.onNext(ocrResultModels); - Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms"); - } else { - Log.e(TAG, "convertImage: failed"); - completed = true; - } - } - } - // 释放资源 - image.close(); + Log.e(TAG, "startScreenCapture: image width = " + image.getWidth()); + Log.e(TAG, "startScreenCapture: image height = " + image.getHeight()); +// Image.Plane[] planes = image.getPlanes(); + // 将 Image 转换为 Bitmap + Bitmap bitmap = convertImageToBitmap(image); + // 释放资源 + image.close(); + + // 处理获取到的屏幕图像 + Log.e(TAG, "startScreenCapture: bitmap width = " + bitmap.getWidth()); + Log.e(TAG, "startScreenCapture: bitmap height = " + bitmap.getHeight()); + predictor.setInputImage(bitmap); + if (onRunModel()) { + Bitmap outputImage = predictor.outputImage(); + Log.e(TAG, "startScreenCapture: outputImage width = " + outputImage.getWidth()); + Log.e(TAG, "startScreenCapture: outputImage height = " + outputImage.getHeight()); + ArrayList ocrResultModels = predictor.getOcrResultModels(); + emitter.onNext(ocrResultModels); + Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms"); } else { - Log.e(TAG, "onImage: image is null"); + Log.e(TAG, "convertImage: failed"); completed = true; } } @@ -399,10 +400,9 @@ public class WeAccessibilityService extends AccessibilityService { @Override public void onNext(ArrayList ocrResultModels) { - Log.e(TAG, "onNext: ocrResultList size = " + ocrResultModels.size()); - Log.v(TAG, "onNext: " + ocrResultModels); + Log.e("convertImage", "onNext: ocrResultList size = " + ocrResultModels.size()); + Log.v("convertImage", "onNext: " + ocrResultModels); analysisUi(ocrResultModels); - completed = true; } @Override @@ -419,7 +419,6 @@ public class WeAccessibilityService extends AccessibilityService { }); } - private void analysisUi(ArrayList ocrResultModels) { String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this); Log.e(TAG, "analysisUi: topAppClassName = " + topAppClassName); @@ -462,25 +461,39 @@ public class WeAccessibilityService extends AccessibilityService { if (stringList.contains("通讯录")) {//在通讯录页面 if (contactSize >= 2) { equalsViewTouch(ocrResultModels, topAppClassName, "标签"); + } else { + completed = true; } + } else { + completed = true; } } break; case "com.tencent.mm.plugin.label.ui.ContactLabelManagerUI"://标签页面 if (stringList.contains("通讯录标签")) {//在标签里面 - containsViewTouch(ocrResultModels, topAppClassName, mTagName); + boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, mTagName); + if (!clicked) { + completed = true; // 点击失败时恢复处理 + } + } else { + completed = true; } break; case "com.tencent.mm.ui.mvvm.MvvmContactListUI"://标签联系人列表 if (stringList.contains(mName) && findString(stringList, mTagName)) { - equalsViewTouch(ocrResultModels, topAppClassName, mName); + boolean clicked = equalsViewTouch(ocrResultModels, topAppClassName, mName); + if (!clicked) { + completed = true; // 点击失败时恢复处理 + } } else { //滑动啥的 - scrollDown(); +// scrollDown(); + completed = true; // 滚动后立即允许下一帧处理 } break; case "com.tencent.mm.plugin.profile.ui.ContactInfoUI"://个人信息页面 + case "com.tencent.mm.ui.widget.dialog.w3": if (findString(stringList, mName) && findString(stringList, DIALER_TEXT)) { if (findString(stringList, CALL_TEXT) && findString(stringList, VIDEO_TEXT)) { String callName; @@ -489,15 +502,13 @@ public class WeAccessibilityService extends AccessibilityService { } else { callName = CALL_TEXT; } - boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName); - Log.e(TAG, "analysisUi: successful = " + successful); + boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName); + Log.e(TAG, "analysisUi: clicked = " + clicked); + if (!clicked) { + completed = true; + } } else { - new Handler(Looper.getMainLooper()).postDelayed(new Runnable() { - @Override - public void run() { - containsViewTouch(ocrResultModels, topAppClassName, DIALER_TEXT); - } - }, 2000); + containsViewTouchDelayed(ocrResultModels, topAppClassName, DIALER_TEXT, 1234); } } else if (findString(stringList, CALL_TEXT) || findString(stringList, VIDEO_TEXT)) { String callName; @@ -506,12 +517,13 @@ public class WeAccessibilityService extends AccessibilityService { } else { callName = CALL_TEXT; } - boolean successful = containsViewTouch(ocrResultModels, topAppClassName, callName); - Log.e(TAG, "analysisUi: successful = " + successful); -// if (successful) {//有可能太快了点击不生效 -// Toaster.showLong("拨打完成"); -// releaseDisplay(); -// } + boolean clicked = containsViewTouch(ocrResultModels, topAppClassName, callName); + Log.e(TAG, "analysisUi: clicked = " + clicked); + if (!clicked) { + completed = true; + } + } else { + completed = true; } break; @@ -540,6 +552,7 @@ public class WeAccessibilityService extends AccessibilityService { releaseDisplay(); break; default: + completed = true; // 确保未知界面不阻塞流程 } // if (stringList.contains("微信")) {//在主页 @@ -591,6 +604,112 @@ public class WeAccessibilityService extends AccessibilityService { // } } + private void initAutoAcceptObservable() { + Observable.create(new ObservableOnSubscribe>() { + @Override + public void subscribe(@NonNull ObservableEmitter> emitter) throws Throwable { + mProcessBitmap = new processBitmap() { + @Override + public void onImage(Bitmap bitmap) { + Log.e(TAG, "initAutoAcceptObservable onImage: time = " + System.currentTimeMillis()); + Log.e(TAG, "initAutoAcceptObservable: bitmap width = " + bitmap.getWidth()); + Log.e(TAG, "initAutoAcceptObservable: bitmap height = " + bitmap.getHeight()); + predictor.setInputImage(bitmap); + if (onRunModel()) { + Bitmap outputImage = predictor.outputImage(); + Log.e(TAG, "initAutoAcceptObservable: outputImage width = " + outputImage.getWidth()); + Log.e(TAG, "initAutoAcceptObservable: outputImage height = " + outputImage.getHeight()); + ArrayList ocrResultModels = predictor.getOcrResultModels(); + emitter.onNext(ocrResultModels); + Log.e(TAG, "convertImage: Inference time: " + predictor.inferenceTime() + " ms"); + } else { + Log.e(TAG, "convertImage: failed"); + completed = true; + } + } + }; + } + }).subscribeOn(Schedulers.computation()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe(new Observer>() { + @Override + public void onSubscribe(Disposable d) { + Log.e("convertImage bitmap", "onSubscribe: "); + } + + @Override + public void onNext(ArrayList ocrResultModels) { + Log.e("convertImage bitmap", "onNext: ocrResultList size = " + ocrResultModels.size()); + Log.v("convertImage bitmap", "onNext: " + ocrResultModels); + analysisAutoAccept(ocrResultModels); + } + + @Override + public void onError(Throwable e) { + Log.e("convertImage bitmap", "onError: " + e.getMessage()); + completed = true; + } + + @Override + public void onComplete() { + Log.e("convertImage bitmap", "onComplete: "); + completed = true; + } + }); + } + + private void analysisAutoAccept(ArrayList ocrResultModels) { + String topAppClassName = ForegroundAppUtil.getForegroundActivityName(WeAccessibilityService.this); + Log.e(TAG, "analysisAutoAccept: topAppClassName = " + topAppClassName); + Log.e(TAG, "analysisAutoAccept: mCurrentClassName = " + mCurrentClassName); + Log.e(TAG, "analysisAutoAccept: mCurrentPackageName = " + mCurrentPackageName); + if (!"com.tencent.mm".equals(mCurrentPackageName)) { + completed = true; // 非微信应用，直接允许处理下一张 + return; + } + if (ocrResultModels == null || ocrResultModels.isEmpty()) { + Log.e(TAG, "analysisAutoAccept: ocrResultModels empty"); + completed = true; + return; + } + + List stringList = ocrResultModels.stream().map(new Function() { + @Override + public String apply(OcrResultModel ocrResultModel) { + return ocrResultModel.getLabel(); + } + }).collect(Collectors.toList()); + + switch (topAppClassName) { + case "com.tencent.mm.plugin.voip.ui.VideoActivity": + if (stringList.contains("接听")) {//在标签里面 + Optional ocrResultOptional = ocrResultModels.stream().filter(new Predicate() { + @Override + public boolean test(OcrResultModel ocrResultModel) { + return "接听".equals(ocrResultModel.getLabel()); + } + }).findAny(); + if (ocrResultOptional.isPresent()) { + OcrResultModel ocrResult = ocrResultOptional.get(); + List points = ocrResult.getPoints(); + Log.e(TAG, "analysisAutoAccept: points = " + points); + Point point = getCenterPoint(points); + Log.e(TAG, "analysisAutoAccept: CenterPoint = " + point); + clickByPoint(point.x, point.y - 75, topAppClassName, new Runnable() { + @Override + public void run() { + answered = true; + mCurrentStep = Step.WAITING; + Toast.makeText(WeAccessibilityService.this, "已自动接听视频/语音", Toast.LENGTH_LONG).show(); + } + }); + } + } + break; + default: + } + } + private boolean clickByPoint(int x, int y, String className, Runnable onComplete) { Log.e(TAG, "clickByNode: x = " + x); Log.e(TAG, "clickByNode: y = " + y); @@ -668,6 +787,8 @@ public class WeAccessibilityService extends AccessibilityService { completed = true; } }); + } else { + completed = true; } return false; } @@ -691,6 +812,38 @@ public class WeAccessibilityService extends AccessibilityService { completed = true; } }); + } else { + completed = true; + } + return false; + } + + private boolean containsViewTouchDelayed(ArrayList ocrResultModels, String className, String text, long millisecond) { + Optional ocrResultOptional = ocrResultModels.stream().filter(new Predicate() { + @Override + public boolean test(OcrResultModel ocrResultModel) { + return ocrResultModel.getLabel().contains(text); + } + }).findAny(); + if (ocrResultOptional.isPresent()) { + OcrResultModel ocrResult = ocrResultOptional.get(); + List points = ocrResult.getPoints(); + Log.e(TAG, "analysisUi: points = " + points); + Point point = getCenterPoint(points); + Log.e(TAG, "analysisUi: CenterPoint = " + point); + return clickByPoint(point.x, point.y, className, new Runnable() { + @Override + public void run() { + new Handler().postDelayed(new Runnable() { + @Override + public void run() { + completed = true; + } + }, millisecond); + } + }); + } else { + completed = true; } return false; } @@ -769,6 +922,32 @@ public class WeAccessibilityService extends AccessibilityService { return predictor.isLoaded() && predictor.runModel(1, 0, 1); } + // 获取屏幕截图（包含状态栏和导航栏） + public void takeScreenshot() { + try { + // 获取DisplayMetrics + DisplayMetrics metrics = new DisplayMetrics(); + WindowManager wm = (WindowManager) getSystemService(Context.WINDOW_SERVICE); + wm.getDefaultDisplay().getRealMetrics(metrics); + + // 反射调用SurfaceControl.screenshot() +// Class surfaceControl = Class.forName("android.view.SurfaceControl"); +// Method screenshotMethod = surfaceControl.getMethod("screenshot", int.class, int.class); +// Bitmap bitmap = (Bitmap) screenshotMethod.invoke(null, metrics.widthPixels, metrics.heightPixels); + // 处理旋转状态（如横屏） +// int rotation = wm.getDefaultDisplay().getRotation(); +// if (rotation != Surface.ROTATION_0) { +// Matrix matrix = new Matrix(); +// matrix.postRotate(90 * rotation); +// bitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); +// } + Bitmap bitmap = SurfaceControl.screenshot(new Rect(), metrics.widthPixels, metrics.heightPixels, Surface.ROTATION_0); + mProcessBitmap.onImage(bitmap); + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 1.在微信页面直接找到联系人拨打电话 * 2.在联系人页面找到并拨打 @@ -805,31 +984,34 @@ public class WeAccessibilityService extends AccessibilityService { } } } -// switch (mCurrentStep) { -// case WAITING: -// mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false); -// Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept); -// if (!mAutoAccept) { -// return; -// } -// if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) { -// mCurrentStep = Step.WECHAT_HANDS_FREE; -// Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show(); -// } else { -// mCurrentStep = Step.WAITING; -//// clickAnswer(); -// } -// break; -// case WECHAT_HANDS_FREE: -// handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT); -// break; -// case DIALER_HANDS_FREE: -// if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) { -// dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT); -// } else { -// mCurrentStep = Step.WAITING; -// } -// break; + switch (mCurrentStep) { + case WAITING: + mAutoAccept = mMMKV.decodeBool(CommonConfig.WECHAT_CALL_AUTO_ACCEPT, false); + Log.e(TAG, "_onAccessibilityEvent: mAutoAccept = " + mAutoAccept); + if (!mAutoAccept) { + return; + } + if (stepAnswer(Property.DESCRIPTION, RECEIVE_DESCRIPTION)) { + mCurrentStep = Step.WECHAT_HANDS_FREE; + Toast.makeText(this, "已自动接听视频/语音", Toast.LENGTH_LONG).show(); + answered = true; + } else { + mCurrentStep = Step.WAITING; +// clickAnswer(); +// if (!answered) + takeScreenshot(); + } + break; + case WECHAT_HANDS_FREE: + handsFree(Property.DESCRIPTION, HANDS_FREE_TEXT); + break; + case DIALER_HANDS_FREE: + if (findHandsFree(Property.DESCRIPTION, DIALER_HANDS_FREE_CLOSE_TEXT)) { + dialerHandsFree(Property.TEXT, DIALER_HANDS_FREE_TEXT); + } else { + mCurrentStep = Step.WAITING; + } + break; // case CLICK_HOME://主页能找到直接点击进去更多 // stepHome(Property.TEXT, mName); // break; @@ -876,7 +1058,7 @@ public class WeAccessibilityService extends AccessibilityService { //// } //// break; // default: -// } + } } @Override @@ -1407,6 +1589,8 @@ public class WeAccessibilityService extends AccessibilityService { if (STOP_RECORD_ACTION.equals(intent.getAction())) { mCurrentPackageName = ""; mCurrentClassName = ""; + completed = true; + answered = false; releaseDisplay(); } }