diff --git a/.gitignore b/.gitignore
index 6cd93c71..4461492c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ sync.ffs_db
/build/
/libs/**/build/
/modules/**/build/
+/plugin-api/**/build/
/captures/
/release/
/releases/
diff --git a/app/build.gradle.kts b/app/build.gradle.kts
index b1155d1d..aa7dcb72 100644
--- a/app/build.gradle.kts
+++ b/app/build.gradle.kts
@@ -166,9 +166,6 @@ dependencies /* Unclassified */ {
// OpenCV
implementation(project(":libs:org-opencv-4_8_0"))
- // PaddleOCR
- implementation(project(":libs:paddleocr"))
-
// RapidOCR
implementation(project(":libs:rapidocr"))
@@ -252,6 +249,9 @@ dependencies /* Unclassified */ {
// ICU4J
implementation(libs.icu4j)
+
+ // Plugin API: Paddle OCR
+ implementation(project(":plugin-api:paddle-ocr"))
}
dependencies /* MIME */ {
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 044786ea..adb1d02e 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -126,6 +126,10 @@
android:protectionLevel="signature"
tools:ignore="ProtectedPermissions" />
+
+
diff --git a/app/src/main/java/org/autojs/autojs/core/plugin/ocr/PaddleOcrPluginHost.kt b/app/src/main/java/org/autojs/autojs/core/plugin/ocr/PaddleOcrPluginHost.kt
new file mode 100644
index 00000000..871eabd5
--- /dev/null
+++ b/app/src/main/java/org/autojs/autojs/core/plugin/ocr/PaddleOcrPluginHost.kt
@@ -0,0 +1,217 @@
+package org.autojs.autojs.core.plugin.ocr
+
+import android.content.ComponentName
+import android.content.Context
+import android.content.Intent
+import android.content.ServiceConnection
+import android.content.pm.PackageManager
+import android.content.pm.ServiceInfo
+import android.graphics.Bitmap
+import android.graphics.Bitmap.CompressFormat
+import android.os.Build
+import android.os.Handler
+import android.os.IBinder
+import android.os.ParcelFileDescriptor
+import android.os.SystemClock.uptimeMillis
+import android.util.Log
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.withContext
+import org.autojs.plugin.paddle.ocr.IOcrPlugin
+import org.autojs.plugin.paddle.ocr.OcrOptions
+import org.autojs.plugin.paddle.ocr.OcrResult
+import org.autojs.plugin.paddle.ocr.PluginInfo
+import java.io.File
+import java.io.FileOutputStream
+
+object PaddleOcrPluginHost {
+
+ private const val TAG = "PaddleOcrPluginHost"
+
+ const val ACTION_OCR = "org.autojs.plugin.PADDLE_OCR"
+
+ private const val DEFAULT_BIND_TIMEOUT_MS = 60_000L
+ private const val DEFAULT_CALL_TIMEOUT_MS = 60_000L
+
+ data class Discovered(
+ val serviceInfo: ServiceInfo,
+ val pluginInfo: PluginInfo?,
+ )
+
+ suspend fun discover(context: Context): List {
+ val pm = context.packageManager
+ val resolveList = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+ pm.queryIntentServices(Intent(ACTION_OCR), PackageManager.ResolveInfoFlags.of(0))
+ } else {
+ pm.queryIntentServices(Intent(ACTION_OCR), 0)
+ }
+ val services = resolveList.mapNotNull { it.serviceInfo }
+ Log.i(TAG, "discover: services=${services.size}")
+ return services.map { svc ->
+ Log.i(TAG, "discover: ${svc.packageName}/${svc.name}")
+ val info = runCatching { withService(context, svc, DEFAULT_BIND_TIMEOUT_MS) { it.getInfo() } }
+ .onFailure { e -> Log.w(TAG, "getInfo failed: ${e.message}") }
+ .getOrNull()
+ Discovered(svc, info)
+ }
+ }
+
+ suspend fun recognizeText(
+ context: Context,
+ target: Discovered,
+ bitmap: Bitmap,
+ options: OcrOptions = OcrOptions(),
+ callTimeoutMs: Long = DEFAULT_CALL_TIMEOUT_MS,
+ ): List {
+ val start = uptimeMillis()
+ return createTempPfd(context, bitmap).use { pfd ->
+ withService(context, target.serviceInfo, DEFAULT_BIND_TIMEOUT_MS) { proxy ->
+ val remain = callTimeoutMs - (uptimeMillis() - start)
+ if (remain <= 0) error("AIDL call timeout in ${callTimeoutMs / 1000} seconds")
+ val lines = proxy.recognizeText(pfd, options)
+ Log.i(TAG, "recognizeTextAidl: got ${lines.size} lines")
+ lines
+ }
+ }
+ }
+
+ suspend fun detect(
+ context: Context,
+ target: Discovered,
+ bitmap: Bitmap,
+ options: OcrOptions = OcrOptions(),
+ callTimeoutMs: Long = DEFAULT_CALL_TIMEOUT_MS,
+ ): List {
+ val start = uptimeMillis()
+ return createTempPfd(context, bitmap).use { pfd ->
+ withService(context, target.serviceInfo, DEFAULT_BIND_TIMEOUT_MS) { proxy ->
+ val remain = callTimeoutMs - (uptimeMillis() - start)
+ if (remain <= 0) error("AIDL call timeout")
+ val results = proxy.detect(pfd, options)
+ Log.i(TAG, "detectAidl: got ${results.size} items")
+ results
+ }
+ }
+ }
+
+ suspend fun select(
+ context: Context,
+ // e.g. "paddle-ocr-v5"
+ engineId: String? = null,
+ // e.g. "paddle-ocr"
+ engine: String? = null,
+ // e.g. "v5"
+ variant: String? = null,
+ ): Discovered? {
+ val list = discover(context).filter { it.pluginInfo != null }
+ if (list.isEmpty()) return null
+ if (engineId != null) {
+ list.firstOrNull { d -> d.pluginInfo?.id == engineId }?.let { return it }
+ }
+ if (engine != null && variant != null) {
+ list.firstOrNull { d -> d.pluginInfo?.engine == engine && d.pluginInfo.variant == variant }?.let { return it }
+ }
+ if (engine != null) {
+ list.firstOrNull { d -> d.pluginInfo?.engine == engine }?.let { return it }
+ }
+ return list.first()
+ }
+
+ // Convert temporary file to read-only FD.
+ // zh-CN: 临时文件转换为只读 FD.
+ private fun createTempPfd(context: Context, bmp: Bitmap): ParcelFileDescriptor {
+ val dir = File(context.cacheDir, "ocr_ipc").apply { if (!exists()) mkdirs() }
+ val f = File.createTempFile("img_", ".bin", dir)
+ FileOutputStream(f).use { fos ->
+ val format = when {
+ bmp.hasAlpha() -> CompressFormat.PNG
+ else -> CompressFormat.JPEG
+ }
+ require(bmp.compress(format, 100, fos)) { "Failed to encode bitmap" }
+ }
+ return ParcelFileDescriptor.open(f, ParcelFileDescriptor.MODE_READ_ONLY)
+ }
+
+ private suspend fun withService(
+ context: Context,
+ serviceInfo: ServiceInfo,
+ bindTimeoutMs: Long = DEFAULT_BIND_TIMEOUT_MS,
+ block: suspend (IOcrPlugin) -> T,
+ ): T = suspendCancellableCoroutine { cont ->
+ val cn = ComponentName(serviceInfo.packageName, serviceInfo.name)
+ val intent = Intent().setComponent(cn)
+ val appCtx = context.applicationContext
+
+ var resolved = false
+ var jobRef: Job? = null
+
+ val conn = object : ServiceConnection {
+ override fun onServiceConnected(name: ComponentName, binder: IBinder) {
+ Log.i(TAG, "onServiceConnected: $name")
+ val proxy = IOcrPlugin.Stub.asInterface(binder)
+ val self = this
+
+ jobRef = CoroutineScope(cont.context + Dispatchers.IO).launch {
+ val result = runCatching { block(proxy) }
+ try {
+ resolved = true
+ if (!cont.isCompleted) cont.resumeWith(result)
+ Log.i(TAG, "resume continuation: success=${result.isSuccess}")
+ } finally {
+ withContext(Dispatchers.Main) {
+ runCatching { appCtx.unbindService(self) }
+ Log.i(TAG, "unbindService: $name")
+ }
+ }
+ }
+ }
+
+ override fun onServiceDisconnected(name: ComponentName) {
+ Log.w(TAG, "onServiceDisconnected: $name")
+ }
+ }
+
+ val ok = try {
+ Log.i(TAG, "bindService: $cn")
+ appCtx.bindService(intent, conn, Context.BIND_AUTO_CREATE)
+ } catch (se: SecurityException) {
+ Log.e(TAG, "bindService SecurityException: $cn | ${se.message}")
+ cont.resumeWith(
+ Result.failure(
+ IllegalStateException(
+ "bindService SecurityException: $cn. Please make sure the plugin declares and the Service uses this permission.", se
+ )
+ )
+ )
+ return@suspendCancellableCoroutine
+ }
+ if (!ok) {
+ Log.e(TAG, "bindService failed: $cn")
+ cont.resumeWith(Result.failure(IllegalStateException("bindService failed: $cn")))
+ return@suspendCancellableCoroutine
+ }
+
+ val cancel = Runnable {
+ if (!resolved && !cont.isCompleted) {
+ Log.e(TAG, "bindService timeout: $cn")
+ cont.resumeWith(Result.failure(TimeoutException("bindService timeout: $cn")))
+ runCatching { appCtx.unbindService(conn) }
+ }
+ }
+ val h = Handler(appCtx.mainLooper)
+ h.postDelayed(cancel, bindTimeoutMs)
+
+ cont.invokeOnCancellation {
+ Log.w(TAG, "continuation cancelled: $cn")
+ h.removeCallbacks(cancel)
+ jobRef?.cancel()
+ runCatching { appCtx.unbindService(conn) }
+ }
+ }
+
+ private class TimeoutException(msg: String) : RuntimeException(msg)
+
+}
diff --git a/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt b/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt
index 62d0201c..2460ea5b 100644
--- a/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt
+++ b/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt
@@ -3,6 +3,9 @@ package org.autojs.autojs.runtime
import android.content.Context
import android.os.Build
import android.util.Log
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
import org.autojs.autojs.AutoJs
import org.autojs.autojs.annotation.ScriptInterface
import org.autojs.autojs.annotation.ScriptVariable
@@ -128,6 +131,7 @@ import java.io.PrintWriter
import java.io.StringWriter
import java.lang.ref.WeakReference
import java.util.concurrent.ConcurrentHashMap
+import kotlin.coroutines.cancellation.CancellationException
import org.autojs.autojs.core.accessibility.UiSelector as CoreUiSelector
import org.autojs.autojs.core.crypto.Crypto as CoreCrypto
import org.autojs.autojs.core.image.Colors as CoreColors
@@ -149,7 +153,6 @@ import org.autojs.autojs.runtime.api.Mime as ApiMime
import org.autojs.autojs.runtime.api.Notice as ApiNotice
import org.autojs.autojs.runtime.api.Ocr as ApiOcr
import org.autojs.autojs.runtime.api.OcrMLKit as ApiOcrMLKit
-import org.autojs.autojs.runtime.api.OcrPaddle as ApiOcrPaddle
import org.autojs.autojs.runtime.api.OcrRapid as ApiOcrRapid
import org.autojs.autojs.runtime.api.Plugins as ApiPlugins
import org.autojs.autojs.runtime.api.Recorder as ApiRecorder
@@ -180,6 +183,10 @@ import org.autojs.autojs.runtime.api.augment.util.VersionCodes as UtilVersionCod
@Suppress("unused", "PropertyName", "PrivatePropertyName")
class ScriptRuntime private constructor(builder: Builder) {
+ private val mJob = SupervisorJob()
+ val coroutineScope = CoroutineScope(Dispatchers.Default + mJob)
+ val coroutineContext = coroutineScope.coroutineContext
+
private var mUiHandlerAppContext: Context
private var mRootShell: AbstractShell? = null
private val mProperties = ConcurrentHashMap()
@@ -335,10 +342,6 @@ class ScriptRuntime private constructor(builder: Builder) {
@ScriptVariable
val ocrMLKit: ApiOcrMLKit
- @JvmField
- @ScriptVariable
- val ocrPaddle: ApiOcrPaddle
-
@JvmField
@ScriptVariable
val ocrRapid: ApiOcrRapid
@@ -474,7 +477,6 @@ class ScriptRuntime private constructor(builder: Builder) {
http = ApiHttp()
ocrMLKit = ApiOcrMLKit()
- ocrPaddle = ApiOcrPaddle()
ocrRapid = ApiOcrRapid()
barcode = ApiBarcode()
@@ -579,6 +581,10 @@ class ScriptRuntime private constructor(builder: Builder) {
@Deprecated("ScriptRuntime#stop is deprecated", ReplaceWith("exit()"))
fun stop() = exit()
+ fun cancelScriptJobs() {
+ mJob.cancel(CancellationException(mUiHandlerAppContext.getString(R.string.error_script_is_on_exiting)))
+ }
+
fun onExit() {
Log.d(TAG, "on exit")
this.isExiting = true
@@ -620,7 +626,6 @@ class ScriptRuntime private constructor(builder: Builder) {
ignoresException({ images.releaseScreenCapturer() })
ignoresException({ images.stopScreenCapturerForegroundService() })
ignoresException({ ocrMLKit.release() })
- ignoresException({ ocrPaddle.release() })
ignoresException({ sensors.unregisterAll() })
ignoresException({ timers.recycle() })
ignoresException({ ui.recycle() })
diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/OcrPaddle.java b/app/src/main/java/org/autojs/autojs/runtime/api/OcrPaddle.java
deleted file mode 100644
index ae880228..00000000
--- a/app/src/main/java/org/autojs/autojs/runtime/api/OcrPaddle.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package org.autojs.autojs.runtime.api;
-
-import android.graphics.Bitmap;
-import android.os.Looper;
-import android.util.Log;
-
-import com.baidu.paddle.lite.ocr.OcrResult;
-import com.baidu.paddle.lite.ocr.Predictor;
-
-import org.autojs.autojs.app.GlobalAppContext;
-import org.autojs.autojs.concurrent.VolatileDispose;
-import org.autojs.autojs.core.image.ImageWrapper;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * @author TonyJiangWJ
- * @since 2023-08-06
- */
-public class OcrPaddle {
-
- private final Predictor mPredictor = new Predictor();
-
- public synchronized boolean init(boolean useSlim) {
- if (!mPredictor.isLoaded || useSlim != mPredictor.isUseSlim()) {
- if (Looper.getMainLooper() == Looper.myLooper()) {
- VolatileDispose result = new VolatileDispose<>();
- new Thread(() -> result.setAndNotify(mPredictor.init(GlobalAppContext.get(), useSlim))).start();
- return Boolean.TRUE.equals(result.blockedGet(60_000));
- } else {
- return mPredictor.init(GlobalAppContext.get(), useSlim);
- }
- }
- return mPredictor.isLoaded;
- }
-
- public void release() {
- mPredictor.releaseModel();
- }
-
- public List detect(ImageWrapper image, int cpuThreadNum, boolean useSlim) {
- if (image == null) {
- return Collections.emptyList();
- }
- Bitmap bitmap = image.getBitmap();
- if (bitmap.isRecycled()) {
- return Collections.emptyList();
- }
- if (mPredictor.cpuThreadNum != cpuThreadNum) {
- mPredictor.releaseModel();
- mPredictor.cpuThreadNum = cpuThreadNum;
- }
- init(useSlim);
- return mPredictor.runOcr(bitmap);
- }
-
- public List detect(ImageWrapper image, int cpuThreadNum) {
- return detect(image, cpuThreadNum, true);
- }
-
- public List detect(ImageWrapper image) {
- return detect(image, 4, true);
- }
-
- public List recognizeText(ImageWrapper image, int cpuThreadNum, boolean useSlim) {
- List words_result = detect(image, cpuThreadNum, useSlim);
- Collections.sort(words_result);
- List outputResult = Arrays.asList(new String[words_result.size()]);
- for (int i = 0; i < words_result.size(); i++) {
- outputResult.set(i, words_result.get(i).getLabel());
- // show LOG in Logcat panel
- Log.i("outputResult", outputResult.get(i));
- }
- return outputResult;
- }
-
- public List recognizeText(ImageWrapper image, int cpuThreadNum) {
- return recognizeText(image, cpuThreadNum, true);
- }
-
- public List recognizeText(ImageWrapper image) {
- return recognizeText(image, 4, true);
- }
-
-}
diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/augment/ocr/OcrPaddle.kt b/app/src/main/java/org/autojs/autojs/runtime/api/augment/ocr/OcrPaddle.kt
index a84e0117..bef01a7f 100644
--- a/app/src/main/java/org/autojs/autojs/runtime/api/augment/ocr/OcrPaddle.kt
+++ b/app/src/main/java/org/autojs/autojs/runtime/api/augment/ocr/OcrPaddle.kt
@@ -4,7 +4,7 @@ import kotlinx.coroutines.runBlocking
import org.autojs.autojs.annotation.RhinoRuntimeFunctionInterface
import org.autojs.autojs.apkbuilder.ApkBuilder
import org.autojs.autojs.core.image.ImageWrapper
-import org.autojs.autojs.core.plugin.ocr.OcrPluginHost
+import org.autojs.autojs.core.plugin.ocr.PaddleOcrPluginHost
import org.autojs.autojs.extension.ScriptableObjectExtensions.inquire
import org.autojs.autojs.runtime.ScriptRuntime
import org.autojs.autojs.runtime.api.OcrResult
@@ -14,6 +14,7 @@ import org.autojs.autojs.runtime.api.augment.ocr.Ocr.Companion.OcrMode
import org.autojs.autojs.runtime.exception.WrappedIllegalArgumentException
import org.autojs.autojs.util.RhinoUtils.coerceBoolean
import org.autojs.autojs.util.RhinoUtils.coerceIntNumber
+import org.autojs.plugin.paddle.ocr.OcrOptions
import org.mozilla.javascript.NativeArray
import org.mozilla.javascript.NativeObject
@@ -48,32 +49,35 @@ class OcrPaddle(private val scriptRuntime: ScriptRuntime) : Augmentable(scriptRu
}
fun recognizeTextImpl(scriptRuntime: ScriptRuntime, image: ImageWrapper, options: NativeObject): List {
- return performOcr(scriptRuntime, image, options).map {
- it.text
- }
- }
-
- fun detectImpl(scriptRuntime: ScriptRuntime, image: ImageWrapper, options: NativeObject): List {
- return performOcr(scriptRuntime, image, options).map {
- OcrResult(it.text, it.confidence, it.bounds)
- }
- }
-
- private fun performOcr(scriptRuntime: ScriptRuntime, image: ImageWrapper, options: NativeObject): List {
ApkBuilder.Libs.PADDLE_OCR.ensureLibFiles(OcrMode.PADDLE.value)
val (cpuThreadNum, useSlim, useOpenCL) = getOcrOptions(options)
- val ocrOptions = org.autojs.plugin.ocr.OcrOptions().apply {
- this.threads = cpuThreadNum
+ val ocrOptions = OcrOptions().apply {
+ this.cpuThreadNum = cpuThreadNum
this.useSlim = useSlim
this.useOpenCL = useOpenCL
}
return runBlocking(scriptRuntime.coroutineContext) {
- val target = OcrPluginHost.select(globalContext)
+ val target = PaddleOcrPluginHost.select(globalContext)
?: throw WrappedIllegalArgumentException("No Paddle OCR plugin matched")
- OcrPluginHost.detect(globalContext, target, image.bitmap, ocrOptions)
+ PaddleOcrPluginHost.recognizeText(globalContext, target, image.bitmap, ocrOptions)
}
}
+ fun detectImpl(scriptRuntime: ScriptRuntime, image: ImageWrapper, options: NativeObject): List {
+ ApkBuilder.Libs.PADDLE_OCR.ensureLibFiles(OcrMode.PADDLE.value)
+ val (cpuThreadNum, useSlim, useOpenCL) = getOcrOptions(options)
+ val ocrOptions = OcrOptions().apply {
+ this.cpuThreadNum = cpuThreadNum
+ this.useSlim = useSlim
+ this.useOpenCL = useOpenCL
+ }
+ return runBlocking(scriptRuntime.coroutineContext) {
+ val target = PaddleOcrPluginHost.select(globalContext)
+ ?: throw WrappedIllegalArgumentException("No Paddle OCR plugin matched")
+ PaddleOcrPluginHost.detect(globalContext, target, image.bitmap, ocrOptions)
+ }.map { OcrResult(it.text, it.confidence, it.bounds) }
+ }
+
private fun getOcrOptions(options: NativeObject): OcrOptions {
val cpuThreadNum = options.inquire("cpuThreadNum", ::coerceIntNumber, DEFAULT_CPU_THREAD_NUM)
val useSlim = options.inquire("useSlim", ::coerceBoolean, DEFAULT_USE_SLIM)
diff --git a/libs/paddleocr/.gitignore b/libs/paddleocr/.gitignore
deleted file mode 100644
index d6443fc1..00000000
--- a/libs/paddleocr/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-/build/
-/.cxx/
-/.idea/
-/cache/
-
-/src/sdk/native/
-
-*.skip
-**/temp-extracted/
diff --git a/libs/paddleocr/PaddleLite/cxx/include/paddle_api.h b/libs/paddleocr/PaddleLite/cxx/include/paddle_api.h
deleted file mode 100644
index 07a23d77..00000000
--- a/libs/paddleocr/PaddleLite/cxx/include/paddle_api.h
+++ /dev/null
@@ -1,611 +0,0 @@
-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/*
- * This file defines PaddlePredictor, the api for lite. It supports multiple
- * hardware including ARM, X86, OpenCL, CUDA and so on.
- */
-
-#ifndef PADDLE_LITE_API_H_ // NOLINT
-#define PADDLE_LITE_API_H_
-#include
-#include