Merge pull request #120 from TonyJiangWJ/master

添加 PaddleOCR 功能 by TonyJiangWJ

app/src/main/assets/init.js

@@ -485,7 +485,7 @@ Object.assign(this, {
 
                 /* ! ocr < images */
                 /* ! ocr < files */
-                [ 'ocr' ],
+                [ 'ocr', 'paddle_ocr' ],
 
                 /* Safe to put last regardless of the order, no guarantee ;). */
                 [ 'floaty', 'storages', 'device', 'recorder', 'toast' ],

app/src/main/assets/modules/__paddle_ocr__.js

@@ -0,0 +1,120 @@
+// noinspection UnnecessaryLocalVariableJS,JSUnusedLocalSymbols
+
+/* Overwritten protection. */
+
+let { images } = global;
+
+/**
+ * @param {org.autojs.autojs.runtime.ScriptRuntime} scriptRuntime
+ * @param {org.mozilla.javascript.Scriptable | global} scope
+ * @return {Internal.Ocr}
+ */
+module.exports = function (scriptRuntime, scope) {
+
+    const rtOcr = scriptRuntime.paddleOCR;
+
+    let _ = {
+        OcrCtor: (/* @IIFE */ () => {
+            /**
+             * @implements Internal.Ocr
+             */
+            const OcrCtor = function () {
+
+                /** @global */
+                const ocr = function (img, options) {
+                    if (typeof arguments[0] === 'string') {
+                        let img = images.read(/* path = */ arguments[0]);
+                        if (img === null) {
+                            throw TypeError(`Invalid image of path "${arguments[0]}" for ocr(img, options?)`);
+                        }
+                        return OcrCtor.prototype.recognizeText(img.oneShot(), options);
+                    }
+                    return OcrCtor.prototype.recognizeText.apply(OcrCtor.prototype, arguments);
+                };
+
+                return Object.assign(ocr, OcrCtor.prototype);
+            };
+
+            OcrCtor.prototype = {
+                constructor: OcrCtor,
+                recognizeText(img, options) {
+                    if (typeof arguments[0] === 'string') {
+                        let img = images.read(/* path = */ arguments[0]);
+                        if (img === null) {
+                            throw TypeError(`Invalid image of path "${arguments[0]}" for ocr.recognizeText(img, options?)`);
+                        }
+                        return this.recognizeText(img.oneShot(), options);
+                    }
+                    if (_.shouldTakenAsRegion(arguments[1])) {
+                        return this.recognizeText(img, { region: arguments[1] });
+                    }
+                    let opt = options || {};
+                    let region = opt.region;
+                    if (region === null) {
+                        return [];
+                    }
+                    let cpuThreadNum = opt.cpuThreadNum || 4
+                    let useSlim = opt.useSlim
+                    if (useSlim === undefined) {
+                       // 默认使用轻量化模型
+                       useSlim = true
+                    }
+                    if (region === undefined) {
+                        return Array.from(rtOcr.recognizeText(img, cpuThreadNum, useSlim));
+                    }
+                    let results = Array.from(rtOcr.recognizeText(images.clip(img, region).oneShot()));
+                    img.shoot();
+                    return results;
+                },
+                detect(img, options) {
+                    if (typeof arguments[0] === 'string') {
+                        let img = images.read(/* path = */ arguments[0]);
+                        if (img === null) {
+                            throw TypeError(`Invalid image of path "${arguments[0]}" for ocr.detect(img, options?)`);
+                        }
+                        return this.detect(img.oneShot(), options);
+                    }
+                    if (_.shouldTakenAsRegion(arguments[1])) {
+                        return this.detect(img, { region: arguments[1] });
+                    }
+                    let opt = options || {};
+                    let region = opt.region;
+                    if (region === null) {
+                        return [];
+                    }
+                    let cpuThreadNum = opt.cpuThreadNum || 4
+                    let useSlim = opt.useSlim
+                    if (useSlim === undefined) {
+                       // 默认使用轻量化模型
+                       useSlim = true
+                    }
+                    /**
+                     * @type {org.autojs.autojs.runtime.api.OcrResult[]}
+                     */
+                    let resultList = rtOcr.detect(region !== undefined ? images.clip(img, region).oneShot() : img, cpuThreadNum, useSlim).toArray();
+                    if (!isNullish(region)) {
+                        resultList.forEach((result) => {
+                            let rect = images.buildRegion(img, region);
+                            result.bounds.offset(rect.x, rect.y);
+                        });
+                    }
+                    return Array.from(resultList);
+                },
+            };
+
+            return OcrCtor;
+        })(),
+        shouldTakenAsRegion(o) {
+            return o instanceof org.opencv.core.Rect
+                || o instanceof android.graphics.Rect
+                || Array.isArray(o);
+        },
+    };
+
+    /**
+     * @type {Internal.Ocr}
+     */
+    const ocr = new _.OcrCtor();
+
+    return ocr;
+};

app/src/main/assets/sample/OCR/PaddleOCR内置API.js

@@ -0,0 +1,15 @@
+/**
+ * @author TonyJiangWJ
+ */
+console.show()
+// 指定是否用精简版模型 速度较快 默认为true
+let useSlim = false
+// cpu线程数量，实际好像没啥作用
+let cpuThreadNum = 4
+let start = new Date()
+let img = images.read('test.png')
+let results = paddle_ocr.detect(img, cpuThreadNum, useSlim)
+toastLog('识别结束, 耗时：' + (new Date() - start) + 'ms')
+log('识别结果：' + JSON.stringify(Array.from(results).map(result => ({ label: result.label, confidence: result.confidence, bounds: result.bounds }))))
+// 回收图片
+img.recycle()

app/src/main/assets/sample/OCR/PaddleOCR原始类.js

@@ -0,0 +1,39 @@
+/**
+ * @author TonyJiangWJ
+ */
+importClass(com.baidu.paddle.lite.ocr.Predictor)
+
+console.show()
+// 指定是否用精简版模型 速度较快
+let useSlim = false
+// 创建检测器
+let predictor = new Predictor()
+// predictor.cpuThreadNum = 4 //可以自定义使用CPU的线程数
+// predictor.checkModelLoaded = false // 可以自定义是否需要校验模型是否成功加载 默认开启 使用内置Base64图片进行校验 识别测试文本来校验模型是否加载成功
+// 初始化模型 首次运行时会比较耗时
+let loading = threads.disposable()
+// 建议在新线程中初始化模型
+threads.start(function () {
+  loading.setAndNotify(predictor.init(context, useSlim))
+  // loading.setAndNotify(predictor.init(context)) 为默认不使用精简版
+  // 内置默认 modelPath 为 models/ocr_v3_for_cpu，初始化自定义模型请写绝对路径否则无法获取到
+  // 内置默认 labelPath 为 labels/ppocr_keys_v1.txt
+  // let modelPath = files.path('./models/customize') // 指定自定义模型路径
+  // let labelPath = files.path('./models/customize') // 指定自定义label路径
+  // 使用自定义模型时det rec cls三个模型文件名称需要手动指定
+  // predictor.detModelFilename = 'det_opt.nb'
+  // predictor.recModelFilename = 'rec_opt.nb'
+  // predictor.clsModelFilename = 'cls_opt.nb'
+  // loading.setAndNotify(predictor.init(context, modelPath, labelPath))
+})
+let loadSuccess = loading.blockedGet()
+toastLog('加载模型结果：' + loadSuccess)
+let start = new Date()
+let img = images.read('test.png')
+let results = predictor.runOcr(img.getBitmap())
+toastLog('识别结束, 耗时：' + (new Date() - start) + 'ms')
+log('识别结果：' + JSON.stringify(Array.from(results).map(result => ({ label: result.label, confidence: result.confidence, bounds: result.bounds }))))
+// 释放模型 用于释放native内存 非必需
+// predictor.releaseModel()
+// 回收图片
+img.recycle()

app/src/main/assets/sample/OCR/PaddleOCR截图识别.js

@@ -0,0 +1,197 @@
+/**
+ * create by TonyJiangWJ
+ */
+let currentEngine = engines.myEngine()
+let runningEngines = engines.all()
+let currentSource = currentEngine.getSource() + ''
+if (runningEngines.length > 1) {
+  runningEngines.forEach(compareEngine => {
+    let compareSource = compareEngine.getSource() + ''
+    if (currentEngine.id !== compareEngine.id && compareSource === currentSource) {
+      // 强制关闭同名的脚本
+      compareEngine.forceStop()
+    }
+  })
+}
+
+
+if (!requestScreenCapture()) {
+  toastLog('请求截图权限失败')
+  exit()
+}
+
+sleep(1000)
+
+importClass(com.baidu.paddle.lite.ocr.Predictor)
+
+// 指定是否用精简版模型 速度较快
+let useSlim = true
+// 创建检测器
+let predictor = new Predictor()
+// predictor.cpuThreadNum = 4 // 可以自定义使用CPU的线程数
+// predictor.checkModelLoaded = false // 可以自定义是否需要校验模型是否成功加载 默认开启 使用内置Base64图片进行校验 识别测试文本来校验模型是否加载成功
+// 初始化模型 首次运行时会比较耗时
+let loading = threads.disposable()
+// 建议在新线程中初始化模型
+threads.start(function () {
+  loading.setAndNotify(predictor.init(context, useSlim))
+  // loading.setAndNotify(predictor.init(context)) 为默认不使用精简版
+  // 内置默认 modelPath 为 models/ocr_v3_for_cpu，初始化自定义模型请写绝对路径否则无法获取到
+  // 内置默认 labelPath 为 labels/ppocr_keys_v1.txt
+  // let modelPath = files.path('./models/customize') // 指定自定义模型路径
+  // let labelPath = files.path('./models/customize') // 指定自定义label路径
+  // 使用自定义模型时det rec cls三个模型文件名称需要手动指定
+  // predictor.detModelFilename = 'det_opt.nb'
+  // predictor.recModelFilename = 'rec_opt.nb'
+  // predictor.clsModelFilename = 'cls_opt.nb'
+  // loading.setAndNotify(predictor.init(context, modelPath, labelPath))
+})
+let loadSuccess = loading.blockedGet()
+if (!loadSuccess) {
+  toastLog('初始化ocr失败')
+  exit()
+}
+// 识别结果和截图信息
+let result = []
+let img = null
+let running = true
+let capturing = true
+
+/**
+ * 截图并识别OCR文本信息
+ */
+function captureAndOcr() {
+  capturing = true
+  img && img.recycle()
+  img = captureScreen()
+  if (!img) {
+    toastLog('截图失败')
+  }
+  let start = new Date()
+  result = predictor.runOcr(img.getBitmap())
+  toastLog('耗时' + (new Date() - start) + 'ms')
+  capturing = false
+}
+
+captureAndOcr()
+
+// 获取状态栏高度
+let offset = -getStatusBarHeightCompat()
+
+// 绘制识别结果
+let window = floaty.rawWindow(
+  <canvas id="canvas" layout_weight="1" />
+);
+
+// 设置悬浮窗位置
+ui.post(() => {
+  window.setPosition(0, offset)
+  window.setSize(device.width, device.height)
+  window.setTouchable(false)
+})
+
+// 操作按钮
+let clickButtonWindow = floaty.rawWindow(
+  <vertical>
+    <button id="captureAndOcr" text="截图识别" />
+    <button id="closeBtn" text="退出" />
+  </vertical>
+);
+ui.run(function () {
+  clickButtonWindow.setPosition(device.width / 2 - ~~(clickButtonWindow.getWidth() / 2), device.height * 0.65)
+})
+
+// 点击识别
+clickButtonWindow.captureAndOcr.click(function () {
+  result = []
+  ui.run(function () {
+    clickButtonWindow.setPosition(device.width, device.height)
+  })
+  setTimeout(() => {
+    captureAndOcr()
+    ui.run(function () {
+      clickButtonWindow.setPosition(device.width / 2 - ~~(clickButtonWindow.getWidth() / 2), device.height * 0.65)
+    })
+  }, 500)
+})
+
+// 点击关闭
+clickButtonWindow.closeBtn.click(function () {
+  exit()
+})
+
+let Typeface = android.graphics.Typeface
+let paint = new Paint()
+paint.setStrokeWidth(1)
+paint.setTypeface(Typeface.DEFAULT_BOLD)
+paint.setTextAlign(Paint.Align.LEFT)
+paint.setAntiAlias(true)
+paint.setStrokeJoin(Paint.Join.ROUND)
+paint.setDither(true)
+window.canvas.on('draw', function (canvas) {
+  if (!running || capturing) {
+    return
+  }
+  // 清空内容
+  canvas.drawColor(0xFFFFFF, android.graphics.PorterDuff.Mode.CLEAR)
+  if (result && result.length > 0) {
+    for (let i = 0; i < result.length; i++) {
+      let ocrResult = result[i]
+      drawRectAndText(ocrResult.label, ocrResult.bounds, '#00ff00', canvas, paint)
+    }
+  }
+})
+
+setInterval(() => { }, 10000)
+events.on('exit', () => {
+  // 标记停止 避免canvas导致闪退
+  running = false
+  // 撤销监听
+  window.canvas.removeAllListeners()
+  // 回收图片
+  img && img.recycle()
+})
+
+/**
+ * 绘制文本和方框
+ *
+ * @param {*} desc
+ * @param {*} rect
+ * @param {*} colorStr
+ * @param {*} canvas
+ * @param {*} paint
+ */
+function drawRectAndText (desc, rect, colorStr, canvas, paint) {
+  let color = colors.parseColor(colorStr)
+
+  paint.setStrokeWidth(1)
+  paint.setStyle(Paint.Style.STROKE)
+  // 反色
+  paint.setARGB(255, 255 - (color >> 16 & 0xff), 255 - (color >> 8 & 0xff), 255 - (color & 0xff))
+  canvas.drawRect(rect, paint)
+  paint.setARGB(255, color >> 16 & 0xff, color >> 8 & 0xff, color & 0xff)
+  paint.setStrokeWidth(1)
+  paint.setTextSize(20)
+  paint.setStyle(Paint.Style.FILL)
+  canvas.drawText(desc, rect.left, rect.top, paint)
+  paint.setTextSize(10)
+  paint.setStrokeWidth(1)
+  paint.setARGB(255, 0, 0, 0)
+}
+
+/**
+ * 获取状态栏高度
+ *
+ * @returns
+ */
+function getStatusBarHeightCompat () {
+  let result = 0
+  let resId = context.getResources().getIdentifier("status_bar_height", "dimen", "android")
+  if (resId > 0) {
+    result = context.getResources().getDimensionPixelOffset(resId)
+  }
+  if (result <= 0) {
+    result = context.getResources().getDimensionPixelOffset(R.dimen.dimen_25dp)
+  }
+  return result
+}

app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.java

@@ -40,6 +40,7 @@ import org.autojs.autojs.runtime.api.Floaty;
 import org.autojs.autojs.runtime.api.Images;
 import org.autojs.autojs.runtime.api.Media;
 import org.autojs.autojs.runtime.api.MlKitOCR;
+import org.autojs.autojs.runtime.api.PaddleOCR;
 import org.autojs.autojs.runtime.api.Plugins;
 import org.autojs.autojs.runtime.api.ProcessShell;
 import org.autojs.autojs.runtime.api.ScreenMetrics;
@@ -243,6 +244,9 @@ public class ScriptRuntime {
     @ScriptVariable
     public final MlKitOCR mlKitOCR;
 
+    @ScriptVariable
+    public final PaddleOCR paddleOCR;
+
     private static WeakReference<Context> applicationContext;
     private final Map<String, Object> mProperties = new ConcurrentHashMap<>();
     private AbstractShell mRootShell;
@@ -278,6 +282,7 @@ public class ScriptRuntime {
         plugins = new Plugins(context, this);
 
         mlKitOCR = new MlKitOCR();
+        paddleOCR = new PaddleOCR();
     }
 
     public void init() {
@@ -554,6 +559,7 @@ public class ScriptRuntime {
         ignoresException(this::recycleShell);
         ignoresException(images::releaseScreenCapturer);
         ignoresException(mlKitOCR::release);
+        ignoresException(paddleOCR::release);
         ignoresException(sensors::unregisterAll);
         ignoresException(timers::recycle);
         ignoresException(ui::recycle);

app/src/main/java/org/autojs/autojs/runtime/api/PaddleOCR.java

@@ -0,0 +1,85 @@
+package org.autojs.autojs.runtime.api;
+
+import android.graphics.Bitmap;
+import android.os.Looper;
+import android.util.Log;
+
+import com.baidu.paddle.lite.ocr.OcrResult;
+import com.baidu.paddle.lite.ocr.Predictor;
+import org.autojs.autojs.app.GlobalAppContext;
+import org.autojs.autojs.concurrent.VolatileDispose;
+import org.autojs.autojs.core.image.ImageWrapper;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author TonyJiangWJ
+ * @since 2023-08-06
+ */
+public class PaddleOCR {
+    private final Predictor mPredictor = new Predictor();
+
+    public synchronized boolean init(boolean useSlim) {
+        if (!mPredictor.isLoaded || useSlim != mPredictor.isUseSlim()) {
+            if (Looper.getMainLooper() == Looper.myLooper()) {
+                VolatileDispose<Boolean> result = new VolatileDispose<>();
+                new Thread(() -> {
+                    result.setAndNotify(mPredictor.init(GlobalAppContext.get(), useSlim));
+                }).start();
+                return result.blockedGet();
+            } else {
+                return mPredictor.init(GlobalAppContext.get(), useSlim);
+            }
+        }
+        return mPredictor.isLoaded;
+    }
+
+    public void release() {
+        mPredictor.releaseModel();
+    }
+
+    public List<OcrResult> detect(ImageWrapper image, int cpuThreadNum, boolean useSlim) {
+        if (image == null) {
+            return Collections.emptyList();
+        }
+        Bitmap bitmap = image.getBitmap();
+        if (bitmap.isRecycled()) {
+            return Collections.emptyList();
+        }
+        if (mPredictor.cpuThreadNum != cpuThreadNum) {
+            mPredictor.releaseModel();
+            mPredictor.cpuThreadNum = cpuThreadNum;
+        }
+        init(useSlim);
+        return mPredictor.runOcr(bitmap);
+    }
+
+    public List<OcrResult> detect(ImageWrapper image, int cpuThreadNum) {
+        return detect(image, cpuThreadNum, true);
+    }
+
+    public List<OcrResult> detect(ImageWrapper image) {
+        return detect(image, 4, true);
+    }
+
+    public String[] recognizeText(ImageWrapper image, int cpuThreadNum, boolean useSlim) {
+        List<OcrResult> words_result = detect(image, cpuThreadNum, useSlim);
+        Collections.sort(words_result);
+        String[] outputResult = new String[words_result.size()];
+        for (int i = 0; i < words_result.size(); i++) {
+            outputResult[i] = words_result.get(i).getLabel();
+            // show LOG in Logcat panel
+            Log.i("outputResult", outputResult[i]);
+        }
+        return outputResult;
+    }
+
+    public String[] recognizeText(ImageWrapper image, int cpuThreadNum) {
+        return recognizeText(image, cpuThreadNum, true);
+    }
+
+    public String[] recognizeText(ImageWrapper image) {
+        return recognizeText(image, 4, true);
+    }
+}

app/src/main/res/raw/licenses.xml

@@ -101,4 +101,10 @@
         <url>http://www.lingala.net/zip4j/</url>
         <license>Apache Software License 2.0</license>
     </notice>
+    <notice>
+        <name>paddle</name>
+        <copyright>Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved</copyright>
+        <url>https://github.com/paddlepaddle/paddle</url>
+        <license>Apache Software License 2.0</license>
+    </notice>
 </notices>