diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 86b9419a..990c250f 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -109,6 +109,9 @@ dependencies /* Unclassified */ { // OpenCV implementation(project(":libs:org.opencv-4.5.5")) + // PaddleOCR + implementation(project(":libs:paddleocr")) + // Android Job implementation(project(":libs:android-job-simplified-1.4.3")) diff --git a/app/src/main/assets/init.js b/app/src/main/assets/init.js index 25b26061..a23e882f 100644 --- a/app/src/main/assets/init.js +++ b/app/src/main/assets/init.js @@ -485,7 +485,7 @@ Object.assign(this, { /* ! ocr < images */ /* ! ocr < files */ - [ 'ocr' ], + [ 'ocr', 'paddle_ocr' ], /* Safe to put last regardless of the order, no guarantee ;). */ [ 'floaty', 'storages', 'device', 'recorder', 'toast' ], diff --git a/app/src/main/assets/modules/__paddle_ocr__.js b/app/src/main/assets/modules/__paddle_ocr__.js new file mode 100644 index 00000000..bab33b74 --- /dev/null +++ b/app/src/main/assets/modules/__paddle_ocr__.js @@ -0,0 +1,120 @@ +// noinspection UnnecessaryLocalVariableJS,JSUnusedLocalSymbols + +/* Overwritten protection. */ + +let { images } = global; + +/** + * @param {org.autojs.autojs.runtime.ScriptRuntime} scriptRuntime + * @param {org.mozilla.javascript.Scriptable | global} scope + * @return {Internal.Ocr} + */ +module.exports = function (scriptRuntime, scope) { + + const rtOcr = scriptRuntime.paddleOCR; + + let _ = { + OcrCtor: (/* @IIFE */ () => { + /** + * @implements Internal.Ocr + */ + const OcrCtor = function () { + + /** @global */ + const ocr = function (img, options) { + if (typeof arguments[0] === 'string') { + let img = images.read(/* path = */ arguments[0]); + if (img === null) { + throw TypeError(`Invalid image of path "${arguments[0]}" for ocr(img, options?)`); + } + return OcrCtor.prototype.recognizeText(img.oneShot(), options); + } + return OcrCtor.prototype.recognizeText.apply(OcrCtor.prototype, arguments); + }; + + return Object.assign(ocr, OcrCtor.prototype); + }; + + OcrCtor.prototype = { + constructor: OcrCtor, + recognizeText(img, options) { + if (typeof arguments[0] === 'string') { + let img = images.read(/* path = */ arguments[0]); + if (img === null) { + throw TypeError(`Invalid image of path "${arguments[0]}" for ocr.recognizeText(img, options?)`); + } + return this.recognizeText(img.oneShot(), options); + } + if (_.shouldTakenAsRegion(arguments[1])) { + return this.recognizeText(img, { region: arguments[1] }); + } + let opt = options || {}; + let region = opt.region; + if (region === null) { + return []; + } + let cpuThreadNum = opt.cpuThreadNum || 4 + let useSlim = opt.useSlim + if (useSlim === undefined) { + // 默认使用轻量化模型 + useSlim = true + } + if (region === undefined) { + return Array.from(rtOcr.recognizeText(img, cpuThreadNum, useSlim)); + } + let results = Array.from(rtOcr.recognizeText(images.clip(img, region).oneShot())); + img.shoot(); + return results; + }, + detect(img, options) { + if (typeof arguments[0] === 'string') { + let img = images.read(/* path = */ arguments[0]); + if (img === null) { + throw TypeError(`Invalid image of path "${arguments[0]}" for ocr.detect(img, options?)`); + } + return this.detect(img.oneShot(), options); + } + if (_.shouldTakenAsRegion(arguments[1])) { + return this.detect(img, { region: arguments[1] }); + } + let opt = options || {}; + let region = opt.region; + if (region === null) { + return []; + } + let cpuThreadNum = opt.cpuThreadNum || 4 + let useSlim = opt.useSlim + if (useSlim === undefined) { + // 默认使用轻量化模型 + useSlim = true + } + /** + * @type {org.autojs.autojs.runtime.api.OcrResult[]} + */ + let resultList = rtOcr.detect(region !== undefined ? images.clip(img, region).oneShot() : img, cpuThreadNum, useSlim).toArray(); + if (!isNullish(region)) { + resultList.forEach((result) => { + let rect = images.buildRegion(img, region); + result.bounds.offset(rect.x, rect.y); + }); + } + return Array.from(resultList); + }, + }; + + return OcrCtor; + })(), + shouldTakenAsRegion(o) { + return o instanceof org.opencv.core.Rect + || o instanceof android.graphics.Rect + || Array.isArray(o); + }, + }; + + /** + * @type {Internal.Ocr} + */ + const ocr = new _.OcrCtor(); + + return ocr; +}; \ No newline at end of file diff --git a/app/src/main/assets/sample/OCR/PaddleOCR内置API.js b/app/src/main/assets/sample/OCR/PaddleOCR内置API.js new file mode 100644 index 00000000..7bd7b8e7 --- /dev/null +++ b/app/src/main/assets/sample/OCR/PaddleOCR内置API.js @@ -0,0 +1,15 @@ +/** + * @author TonyJiangWJ + */ +console.show() +// 指定是否用精简版模型 速度较快 默认为true +let useSlim = false +// cpu线程数量,实际好像没啥作用 +let cpuThreadNum = 4 +let start = new Date() +let img = images.read('test.png') +let results = paddle_ocr.detect(img, cpuThreadNum, useSlim) +toastLog('识别结束, 耗时:' + (new Date() - start) + 'ms') +log('识别结果:' + JSON.stringify(Array.from(results).map(result => ({ label: result.label, confidence: result.confidence, bounds: result.bounds })))) +// 回收图片 +img.recycle() diff --git a/app/src/main/assets/sample/OCR/PaddleOCR原始类.js b/app/src/main/assets/sample/OCR/PaddleOCR原始类.js new file mode 100644 index 00000000..73ee8266 --- /dev/null +++ b/app/src/main/assets/sample/OCR/PaddleOCR原始类.js @@ -0,0 +1,39 @@ +/** + * @author TonyJiangWJ + */ +importClass(com.baidu.paddle.lite.ocr.Predictor) + +console.show() +// 指定是否用精简版模型 速度较快 +let useSlim = false +// 创建检测器 +let predictor = new Predictor() +// predictor.cpuThreadNum = 4 //可以自定义使用CPU的线程数 +// predictor.checkModelLoaded = false // 可以自定义是否需要校验模型是否成功加载 默认开启 使用内置Base64图片进行校验 识别测试文本来校验模型是否加载成功 +// 初始化模型 首次运行时会比较耗时 +let loading = threads.disposable() +// 建议在新线程中初始化模型 +threads.start(function () { + loading.setAndNotify(predictor.init(context, useSlim)) + // loading.setAndNotify(predictor.init(context)) 为默认不使用精简版 + // 内置默认 modelPath 为 models/ocr_v3_for_cpu,初始化自定义模型请写绝对路径否则无法获取到 + // 内置默认 labelPath 为 labels/ppocr_keys_v1.txt + // let modelPath = files.path('./models/customize') // 指定自定义模型路径 + // let labelPath = files.path('./models/customize') // 指定自定义label路径 + // 使用自定义模型时det rec cls三个模型文件名称需要手动指定 + // predictor.detModelFilename = 'det_opt.nb' + // predictor.recModelFilename = 'rec_opt.nb' + // predictor.clsModelFilename = 'cls_opt.nb' + // loading.setAndNotify(predictor.init(context, modelPath, labelPath)) +}) +let loadSuccess = loading.blockedGet() +toastLog('加载模型结果:' + loadSuccess) +let start = new Date() +let img = images.read('test.png') +let results = predictor.runOcr(img.getBitmap()) +toastLog('识别结束, 耗时:' + (new Date() - start) + 'ms') +log('识别结果:' + JSON.stringify(Array.from(results).map(result => ({ label: result.label, confidence: result.confidence, bounds: result.bounds })))) +// 释放模型 用于释放native内存 非必需 +// predictor.releaseModel() +// 回收图片 +img.recycle() diff --git a/app/src/main/assets/sample/OCR/PaddleOCR截图识别.js b/app/src/main/assets/sample/OCR/PaddleOCR截图识别.js new file mode 100644 index 00000000..64557dc3 --- /dev/null +++ b/app/src/main/assets/sample/OCR/PaddleOCR截图识别.js @@ -0,0 +1,197 @@ +/** + * create by TonyJiangWJ + */ +let currentEngine = engines.myEngine() +let runningEngines = engines.all() +let currentSource = currentEngine.getSource() + '' +if (runningEngines.length > 1) { + runningEngines.forEach(compareEngine => { + let compareSource = compareEngine.getSource() + '' + if (currentEngine.id !== compareEngine.id && compareSource === currentSource) { + // 强制关闭同名的脚本 + compareEngine.forceStop() + } + }) +} + + +if (!requestScreenCapture()) { + toastLog('请求截图权限失败') + exit() +} + +sleep(1000) + +importClass(com.baidu.paddle.lite.ocr.Predictor) + +// 指定是否用精简版模型 速度较快 +let useSlim = true +// 创建检测器 +let predictor = new Predictor() +// predictor.cpuThreadNum = 4 // 可以自定义使用CPU的线程数 +// predictor.checkModelLoaded = false // 可以自定义是否需要校验模型是否成功加载 默认开启 使用内置Base64图片进行校验 识别测试文本来校验模型是否加载成功 +// 初始化模型 首次运行时会比较耗时 +let loading = threads.disposable() +// 建议在新线程中初始化模型 +threads.start(function () { + loading.setAndNotify(predictor.init(context, useSlim)) + // loading.setAndNotify(predictor.init(context)) 为默认不使用精简版 + // 内置默认 modelPath 为 models/ocr_v3_for_cpu,初始化自定义模型请写绝对路径否则无法获取到 + // 内置默认 labelPath 为 labels/ppocr_keys_v1.txt + // let modelPath = files.path('./models/customize') // 指定自定义模型路径 + // let labelPath = files.path('./models/customize') // 指定自定义label路径 + // 使用自定义模型时det rec cls三个模型文件名称需要手动指定 + // predictor.detModelFilename = 'det_opt.nb' + // predictor.recModelFilename = 'rec_opt.nb' + // predictor.clsModelFilename = 'cls_opt.nb' + // loading.setAndNotify(predictor.init(context, modelPath, labelPath)) +}) +let loadSuccess = loading.blockedGet() +if (!loadSuccess) { + toastLog('初始化ocr失败') + exit() +} +// 识别结果和截图信息 +let result = [] +let img = null +let running = true +let capturing = true + +/** + * 截图并识别OCR文本信息 + */ +function captureAndOcr() { + capturing = true + img && img.recycle() + img = captureScreen() + if (!img) { + toastLog('截图失败') + } + let start = new Date() + result = predictor.runOcr(img.getBitmap()) + toastLog('耗时' + (new Date() - start) + 'ms') + capturing = false +} + +captureAndOcr() + +// 获取状态栏高度 +let offset = -getStatusBarHeightCompat() + +// 绘制识别结果 +let window = floaty.rawWindow( + +); + +// 设置悬浮窗位置 +ui.post(() => { + window.setPosition(0, offset) + window.setSize(device.width, device.height) + window.setTouchable(false) +}) + +// 操作按钮 +let clickButtonWindow = floaty.rawWindow( + +