添加PaddleOCR功能

This commit is contained in:
TonyJiangWJ
2023-08-06 23:58:21 +08:00
parent 49e9961a86
commit 61f7c13262
65 changed files with 16875 additions and 1 deletions

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
</manifest>

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,93 @@
# For more information about using CMake with Android Studio, read the
# documentation: https://d.android.com/studio/projects/add-native-code.html
# Sets the minimum version of CMake required to build the native library.
cmake_minimum_required(VERSION 3.4.1)
# Creates and names a library, sets it as either STATIC or SHARED, and provides
# the relative paths to its source code. You can define multiple libraries, and
# CMake builds them for you. Gradle automatically packages shared libraries with
# your APK.
set(PaddleLite_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../PaddleLite")
include_directories(${PaddleLite_DIR}/cxx/include)
set(OpenCV_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../OpenCV/OpenCV-android-sdk/sdk/native/jni")
message(STATUS "opencv dir: ${OpenCV_DIR}")
find_package(OpenCV REQUIRED)
message(STATUS "OpenCV libraries: ${OpenCV_LIBS}")
include_directories(${OpenCV_INCLUDE_DIRS})
aux_source_directory(. SOURCES)
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -ffast-math -Ofast -Os"
)
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -fvisibility=hidden -fvisibility-inlines-hidden -fdata-sections -ffunction-sections"
)
set(CMAKE_SHARED_LINKER_FLAGS
"${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,-z,nocopyreloc")
add_library(
# Sets the name of the library.
Native
# Sets the library as a shared library.
SHARED
# Provides a relative path to your source file(s).
${SOURCES})
find_library(
# Sets the name of the path variable.
log-lib
# Specifies the name of the NDK library that you want CMake to locate.
log)
add_library(
# Sets the name of the library.
paddle_light_api_shared
# Sets the library as a shared library.
SHARED
# Provides a relative path to your source file(s).
IMPORTED)
set_target_properties(
# Specifies the target library.
paddle_light_api_shared
# Specifies the parameter you want to define.
PROPERTIES
IMPORTED_LOCATION
${PaddleLite_DIR}/cxx/libs/${ANDROID_ABI}/libpaddle_light_api_shared.so
# Provides the path to the library you want to import.
)
# Specifies libraries CMake should link to your target library. You can link
# multiple libraries, such as libraries you define in this build script,
# prebuilt third-party libraries, or system libraries.
target_link_libraries(
# Specifies the target library.
Native
paddle_light_api_shared
${OpenCV_LIBS}
GLESv2
EGL
jnigraphics
${log-lib}
)
add_custom_command(
TARGET Native
POST_BUILD
COMMAND
${CMAKE_COMMAND} -E copy
${PaddleLite_DIR}/cxx/libs/${ANDROID_ABI}/libc++_shared.so
${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libc++_shared.so)
add_custom_command(
TARGET Native
POST_BUILD
COMMAND
${CMAKE_COMMAND} -E copy
${PaddleLite_DIR}/cxx/libs/${ANDROID_ABI}/libpaddle_light_api_shared.so
${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libpaddle_light_api_shared.so)

View File

@@ -0,0 +1,37 @@
//
// Created by fu on 4/25/18.
//
#pragma once
#import <numeric>
#import <vector>
#ifdef __ANDROID__
#include <android/log.h>
#define LOG_TAG "OCR_NDK"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
#else
#include <stdio.h>
#define LOGI(format, ...) \
fprintf(stdout, "[" LOG_TAG "]" format "\n", ##__VA_ARGS__)
#define LOGW(format, ...) \
fprintf(stdout, "[" LOG_TAG "]" format "\n", ##__VA_ARGS__)
#define LOGE(format, ...) \
fprintf(stderr, "[" LOG_TAG "]Error: " format "\n", ##__VA_ARGS__)
#endif
enum RETURN_CODE { RETURN_OK = 0 };
enum NET_TYPE { NET_OCR = 900100, NET_OCR_INTERNAL = 991008 };
template <typename T> inline T product(const std::vector<T> &vec) {
if (vec.empty()) {
return 0;
}
return std::accumulate(vec.begin(), vec.end(), 1, std::multiplies<T>());
}

View File

@@ -0,0 +1,127 @@
//
// Created by fujiayi on 2020/7/5.
//
#include "native.h"
#include "ocr_ppredictor.h"
#include <algorithm>
#include <paddle_api.h>
#include <string>
static paddle::lite_api::PowerMode str_to_cpu_mode(const std::string &cpu_mode);
extern "C" JNIEXPORT jlong JNICALL
Java_com_baidu_paddle_lite_ocr_OCRPredictorNative_init(
JNIEnv *env, jobject thiz, jstring j_det_model_path,
jstring j_rec_model_path, jstring j_cls_model_path, jint j_use_opencl, jint j_thread_num,
jstring j_cpu_mode) {
std::string det_model_path = jstring_to_cpp_string(env, j_det_model_path);
std::string rec_model_path = jstring_to_cpp_string(env, j_rec_model_path);
std::string cls_model_path = jstring_to_cpp_string(env, j_cls_model_path);
int thread_num = j_thread_num;
std::string cpu_mode = jstring_to_cpp_string(env, j_cpu_mode);
ppredictor::OCR_Config conf;
conf.use_opencl = j_use_opencl;
conf.thread_num = thread_num;
conf.mode = str_to_cpu_mode(cpu_mode);
ppredictor::OCR_PPredictor *orc_predictor =
new ppredictor::OCR_PPredictor{conf};
orc_predictor->init_from_file(det_model_path, rec_model_path, cls_model_path);
return reinterpret_cast<jlong>(orc_predictor);
}
/**
* "LITE_POWER_HIGH" convert to paddle::lite_api::LITE_POWER_HIGH
* @param cpu_mode
* @return
*/
static paddle::lite_api::PowerMode
str_to_cpu_mode(const std::string &cpu_mode) {
static std::map<std::string, paddle::lite_api::PowerMode> cpu_mode_map{
{"LITE_POWER_HIGH", paddle::lite_api::LITE_POWER_HIGH},
{"LITE_POWER_LOW", paddle::lite_api::LITE_POWER_HIGH},
{"LITE_POWER_FULL", paddle::lite_api::LITE_POWER_FULL},
{"LITE_POWER_NO_BIND", paddle::lite_api::LITE_POWER_NO_BIND},
{"LITE_POWER_RAND_HIGH", paddle::lite_api::LITE_POWER_RAND_HIGH},
{"LITE_POWER_RAND_LOW", paddle::lite_api::LITE_POWER_RAND_LOW}};
std::string upper_key;
std::transform(cpu_mode.cbegin(), cpu_mode.cend(), upper_key.begin(),
::toupper);
auto index = cpu_mode_map.find(upper_key.c_str());
if (index == cpu_mode_map.end()) {
// 可能因为大小写转换后找不到 直接通过入参查找
index = cpu_mode_map.find(cpu_mode);
if (index != cpu_mode_map.end()) {
LOGI("find cpu_mode by &cpu_mode %s", cpu_mode.c_str());
return index->second;
}
LOGE("cpu_mode not found %s", upper_key.c_str());
return paddle::lite_api::LITE_POWER_HIGH;
} else {
return index->second;
}
}
extern "C" JNIEXPORT jfloatArray JNICALL
Java_com_baidu_paddle_lite_ocr_OCRPredictorNative_forward(
JNIEnv *env, jobject thiz, jlong java_pointer,
jobject original_image,jint j_max_size_len, jint j_run_det, jint j_run_cls,
jint j_run_rec) {
LOGI("begin to run native forward");
if (java_pointer == 0) {
LOGE("JAVA pointer is NULL");
return cpp_array_to_jfloatarray(env, nullptr, 0);
}
cv::Mat origin = bitmap_to_cv_mat(env, original_image);
if (origin.size == nullptr) {
LOGE("origin bitmap cannot convert to CV Mat");
return cpp_array_to_jfloatarray(env, nullptr, 0);
}
int max_size_len = j_max_size_len;
int run_det = j_run_det;
int run_cls = j_run_cls;
int run_rec = j_run_rec;
ppredictor::OCR_PPredictor *ppredictor =
(ppredictor::OCR_PPredictor *)java_pointer;
std::vector<int64_t> dims_arr;
std::vector<ppredictor::OCRPredictResult> results =
ppredictor->infer_ocr(origin, max_size_len, run_det, run_cls, run_rec);
LOGI("infer_ocr finished with boxes %ld", results.size());
// 这里将std::vector<ppredictor::OCRPredictResult> 序列化成
// float数组,传输到java层再反序列化
std::vector<float> float_arr;
for (const ppredictor::OCRPredictResult &r : results) {
float_arr.push_back(r.points.size());
float_arr.push_back(r.word_index.size());
float_arr.push_back(r.score);
// add det point
for (const std::vector<int> &point : r.points) {
float_arr.push_back(point.at(0));
float_arr.push_back(point.at(1));
}
// add rec word idx
for (int index : r.word_index) {
float_arr.push_back(index);
}
// add cls result
float_arr.push_back(r.cls_label);
float_arr.push_back(r.cls_score);
}
return cpp_array_to_jfloatarray(env, float_arr.data(), float_arr.size());
}
extern "C" JNIEXPORT void JNICALL
Java_com_baidu_paddle_lite_ocr_OCRPredictorNative_release(
JNIEnv *env, jobject thiz, jlong java_pointer) {
if (java_pointer == 0) {
LOGE("JAVA pointer is NULL");
return;
}
ppredictor::OCR_PPredictor *ppredictor =
(ppredictor::OCR_PPredictor *)java_pointer;
delete ppredictor;
}

View File

@@ -0,0 +1,137 @@
//
// Created by fujiayi on 2020/7/5.
//
#pragma once
#include "common.h"
#include <android/bitmap.h>
#include <jni.h>
#include <opencv2/opencv.hpp>
#include <string>
#include <vector>
inline std::string jstring_to_cpp_string(JNIEnv *env, jstring jstr) {
// In java, a unicode char will be encoded using 2 bytes (utf16).
// so jstring will contain characters utf16. std::string in c++ is
// essentially a string of bytes, not characters, so if we want to
// pass jstring from JNI to c++, we have convert utf16 to bytes.
if (!jstr) {
return "";
}
const jclass stringClass = env->GetObjectClass(jstr);
const jmethodID getBytes =
env->GetMethodID(stringClass, "getBytes", "(Ljava/lang/String;)[B");
const jbyteArray stringJbytes = (jbyteArray)env->CallObjectMethod(
jstr, getBytes, env->NewStringUTF("UTF-8"));
size_t length = (size_t)env->GetArrayLength(stringJbytes);
jbyte *pBytes = env->GetByteArrayElements(stringJbytes, NULL);
std::string ret = std::string(reinterpret_cast<char *>(pBytes), length);
env->ReleaseByteArrayElements(stringJbytes, pBytes, JNI_ABORT);
env->DeleteLocalRef(stringJbytes);
env->DeleteLocalRef(stringClass);
return ret;
}
inline jstring cpp_string_to_jstring(JNIEnv *env, std::string str) {
auto *data = str.c_str();
jclass strClass = env->FindClass("java/lang/String");
jmethodID strClassInitMethodID =
env->GetMethodID(strClass, "<init>", "([BLjava/lang/String;)V");
jbyteArray bytes = env->NewByteArray(strlen(data));
env->SetByteArrayRegion(bytes, 0, strlen(data),
reinterpret_cast<const jbyte *>(data));
jstring encoding = env->NewStringUTF("UTF-8");
jstring res = (jstring)(
env->NewObject(strClass, strClassInitMethodID, bytes, encoding));
env->DeleteLocalRef(strClass);
env->DeleteLocalRef(encoding);
env->DeleteLocalRef(bytes);
return res;
}
inline jfloatArray cpp_array_to_jfloatarray(JNIEnv *env, const float *buf,
int64_t len) {
if (len == 0) {
return env->NewFloatArray(0);
}
jfloatArray result = env->NewFloatArray(len);
env->SetFloatArrayRegion(result, 0, len, buf);
return result;
}
inline jintArray cpp_array_to_jintarray(JNIEnv *env, const int *buf,
int64_t len) {
jintArray result = env->NewIntArray(len);
env->SetIntArrayRegion(result, 0, len, buf);
return result;
}
inline jbyteArray cpp_array_to_jbytearray(JNIEnv *env, const int8_t *buf,
int64_t len) {
jbyteArray result = env->NewByteArray(len);
env->SetByteArrayRegion(result, 0, len, buf);
return result;
}
inline jlongArray int64_vector_to_jlongarray(JNIEnv *env,
const std::vector<int64_t> &vec) {
jlongArray result = env->NewLongArray(vec.size());
jlong *buf = new jlong[vec.size()];
for (size_t i = 0; i < vec.size(); ++i) {
buf[i] = (jlong)vec[i];
}
env->SetLongArrayRegion(result, 0, vec.size(), buf);
delete[] buf;
return result;
}
inline std::vector<int64_t> jlongarray_to_int64_vector(JNIEnv *env,
jlongArray data) {
int data_size = env->GetArrayLength(data);
jlong *data_ptr = env->GetLongArrayElements(data, nullptr);
std::vector<int64_t> data_vec(data_ptr, data_ptr + data_size);
env->ReleaseLongArrayElements(data, data_ptr, 0);
return data_vec;
}
inline std::vector<float> jfloatarray_to_float_vector(JNIEnv *env,
jfloatArray data) {
int data_size = env->GetArrayLength(data);
jfloat *data_ptr = env->GetFloatArrayElements(data, nullptr);
std::vector<float> data_vec(data_ptr, data_ptr + data_size);
env->ReleaseFloatArrayElements(data, data_ptr, 0);
return data_vec;
}
inline cv::Mat bitmap_to_cv_mat(JNIEnv *env, jobject bitmap) {
AndroidBitmapInfo info;
int result = AndroidBitmap_getInfo(env, bitmap, &info);
if (result != ANDROID_BITMAP_RESULT_SUCCESS) {
LOGE("AndroidBitmap_getInfo failed, result: %d", result);
return cv::Mat{};
}
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
LOGE("Bitmap format is not RGBA_8888 !");
return cv::Mat{};
}
unsigned char *srcData = NULL;
AndroidBitmap_lockPixels(env, bitmap, (void **)&srcData);
cv::Mat mat = cv::Mat::zeros(info.height, info.width, CV_8UC4);
memcpy(mat.data, srcData, info.height * info.width * 4);
AndroidBitmap_unlockPixels(env, bitmap);
cv::cvtColor(mat, mat, cv::COLOR_RGBA2BGR);
/**
if (!cv::imwrite("/sdcard/1/copy.jpg", mat)){
LOGE("Write image failed " );
}
*/
return mat;
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,544 @@
/*******************************************************************************
* *
* Author : Angus Johnson *
* Version : 6.4.2 *
* Date : 27 February 2017 *
* Website : http://www.angusj.com *
* Copyright : Angus Johnson 2010-2017 *
* *
* License: *
* Use, modification & distribution is subject to Boost Software License Ver 1. *
* http://www.boost.org/LICENSE_1_0.txt *
* *
* Attributions: *
* The code in this library is an extension of Bala Vatti's clipping algorithm: *
* "A generic solution to polygon clipping" *
* Communications of the ACM, Vol 35, Issue 7 (July 1992) pp 56-63. *
* http://portal.acm.org/citation.cfm?id=129906 *
* *
* Computer graphics and geometric modeling: implementation and algorithms *
* By Max K. Agoston *
* Springer; 1 edition (January 4, 2005) *
* http://books.google.com/books?q=vatti+clipping+agoston *
* *
* See also: *
* "Polygon Offsetting by Computing Winding Numbers" *
* Paper no. DETC2005-85513 pp. 565-575 *
* ASME 2005 International Design Engineering Technical Conferences *
* and Computers and Information in Engineering Conference (IDETC/CIE2005) *
* September 24-28, 2005 , Long Beach, California, USA *
* http://www.me.berkeley.edu/~mcmains/pubs/DAC05OffsetPolygon.pdf *
* *
*******************************************************************************/
#ifndef clipper_hpp
#define clipper_hpp
#define CLIPPER_VERSION "6.4.2"
// use_int32: When enabled 32bit ints are used instead of 64bit ints. This
// improve performance but coordinate values are limited to the range +/- 46340
//#define use_int32
// use_xyz: adds a Z member to IntPoint. Adds a minor cost to perfomance.
//#define use_xyz
// use_lines: Enables line clipping. Adds a very minor cost to performance.
#define use_lines
// use_deprecated: Enables temporary support for the obsolete functions
//#define use_deprecated
#include <cstdlib>
#include <cstring>
#include <functional>
#include <list>
#include <ostream>
#include <queue>
#include <set>
#include <stdexcept>
#include <vector>
namespace ClipperLib {
enum ClipType { ctIntersection, ctUnion, ctDifference, ctXor };
enum PolyType { ptSubject, ptClip };
// By far the most widely used winding rules for polygon filling are
// EvenOdd & NonZero (GDI, GDI+, XLib, OpenGL, Cairo, AGG, Quartz, SVG, Gr32)
// Others rules include Positive, Negative and ABS_GTR_EQ_TWO (only in OpenGL)
// see http://glprogramming.com/red/chapter11.html
enum PolyFillType { pftEvenOdd, pftNonZero, pftPositive, pftNegative };
#ifdef use_int32
typedef int cInt;
static cInt const loRange = 0x7FFF;
static cInt const hiRange = 0x7FFF;
#else
typedef signed long long cInt;
static cInt const loRange = 0x3FFFFFFF;
static cInt const hiRange = 0x3FFFFFFFFFFFFFFFLL;
typedef signed long long long64; // used by Int128 class
typedef unsigned long long ulong64;
#endif
struct IntPoint {
cInt X;
cInt Y;
#ifdef use_xyz
cInt Z;
IntPoint(cInt x = 0, cInt y = 0, cInt z = 0) : X(x), Y(y), Z(z){};
#else
IntPoint(cInt x = 0, cInt y = 0) : X(x), Y(y){};
#endif
friend inline bool operator==(const IntPoint &a, const IntPoint &b) {
return a.X == b.X && a.Y == b.Y;
}
friend inline bool operator!=(const IntPoint &a, const IntPoint &b) {
return a.X != b.X || a.Y != b.Y;
}
};
//------------------------------------------------------------------------------
typedef std::vector<IntPoint> Path;
typedef std::vector<Path> Paths;
inline Path &operator<<(Path &poly, const IntPoint &p) {
poly.push_back(p);
return poly;
}
inline Paths &operator<<(Paths &polys, const Path &p) {
polys.push_back(p);
return polys;
}
std::ostream &operator<<(std::ostream &s, const IntPoint &p);
std::ostream &operator<<(std::ostream &s, const Path &p);
std::ostream &operator<<(std::ostream &s, const Paths &p);
struct DoublePoint {
double X;
double Y;
DoublePoint(double x = 0, double y = 0) : X(x), Y(y) {}
DoublePoint(IntPoint ip) : X((double)ip.X), Y((double)ip.Y) {}
};
//------------------------------------------------------------------------------
#ifdef use_xyz
typedef void (*ZFillCallback)(IntPoint &e1bot, IntPoint &e1top, IntPoint &e2bot,
IntPoint &e2top, IntPoint &pt);
#endif
enum InitOptions {
ioReverseSolution = 1,
ioStrictlySimple = 2,
ioPreserveCollinear = 4
};
enum JoinType { jtSquare, jtRound, jtMiter };
enum EndType {
etClosedPolygon,
etClosedLine,
etOpenButt,
etOpenSquare,
etOpenRound
};
class PolyNode;
typedef std::vector<PolyNode *> PolyNodes;
class PolyNode {
public:
PolyNode();
virtual ~PolyNode(){};
Path Contour;
PolyNodes Childs;
PolyNode *Parent;
PolyNode *GetNext() const;
bool IsHole() const;
bool IsOpen() const;
int ChildCount() const;
private:
// PolyNode& operator =(PolyNode& other);
unsigned Index; // node index in Parent.Childs
bool m_IsOpen;
JoinType m_jointype;
EndType m_endtype;
PolyNode *GetNextSiblingUp() const;
void AddChild(PolyNode &child);
friend class Clipper; // to access Index
friend class ClipperOffset;
};
class PolyTree : public PolyNode {
public:
~PolyTree() { Clear(); };
PolyNode *GetFirst() const;
void Clear();
int Total() const;
private:
// PolyTree& operator =(PolyTree& other);
PolyNodes AllNodes;
friend class Clipper; // to access AllNodes
};
bool Orientation(const Path &poly);
double Area(const Path &poly);
int PointInPolygon(const IntPoint &pt, const Path &path);
void SimplifyPolygon(const Path &in_poly, Paths &out_polys,
PolyFillType fillType = pftEvenOdd);
void SimplifyPolygons(const Paths &in_polys, Paths &out_polys,
PolyFillType fillType = pftEvenOdd);
void SimplifyPolygons(Paths &polys, PolyFillType fillType = pftEvenOdd);
void CleanPolygon(const Path &in_poly, Path &out_poly, double distance = 1.415);
void CleanPolygon(Path &poly, double distance = 1.415);
void CleanPolygons(const Paths &in_polys, Paths &out_polys,
double distance = 1.415);
void CleanPolygons(Paths &polys, double distance = 1.415);
void MinkowskiSum(const Path &pattern, const Path &path, Paths &solution,
bool pathIsClosed);
void MinkowskiSum(const Path &pattern, const Paths &paths, Paths &solution,
bool pathIsClosed);
void MinkowskiDiff(const Path &poly1, const Path &poly2, Paths &solution);
void PolyTreeToPaths(const PolyTree &polytree, Paths &paths);
void ClosedPathsFromPolyTree(const PolyTree &polytree, Paths &paths);
void OpenPathsFromPolyTree(PolyTree &polytree, Paths &paths);
void ReversePath(Path &p);
void ReversePaths(Paths &p);
struct IntRect {
cInt left;
cInt top;
cInt right;
cInt bottom;
};
// enums that are used internally ...
enum EdgeSide { esLeft = 1, esRight = 2 };
// forward declarations (for stuff used internally) ...
struct TEdge;
struct IntersectNode;
struct LocalMinimum;
struct OutPt;
struct OutRec;
struct Join;
typedef std::vector<OutRec *> PolyOutList;
typedef std::vector<TEdge *> EdgeList;
typedef std::vector<Join *> JoinList;
typedef std::vector<IntersectNode *> IntersectList;
//------------------------------------------------------------------------------
// ClipperBase is the ancestor to the Clipper class. It should not be
// instantiated directly. This class simply abstracts the conversion of sets of
// polygon coordinates into edge objects that are stored in a LocalMinima list.
class ClipperBase {
public:
ClipperBase();
virtual ~ClipperBase();
virtual bool AddPath(const Path &pg, PolyType PolyTyp, bool Closed);
bool AddPaths(const Paths &ppg, PolyType PolyTyp, bool Closed);
virtual void Clear();
IntRect GetBounds();
bool PreserveCollinear() { return m_PreserveCollinear; };
void PreserveCollinear(bool value) { m_PreserveCollinear = value; };
protected:
void DisposeLocalMinimaList();
TEdge *AddBoundsToLML(TEdge *e, bool IsClosed);
virtual void Reset();
TEdge *ProcessBound(TEdge *E, bool IsClockwise);
void InsertScanbeam(const cInt Y);
bool PopScanbeam(cInt &Y);
bool LocalMinimaPending();
bool PopLocalMinima(cInt Y, const LocalMinimum *&locMin);
OutRec *CreateOutRec();
void DisposeAllOutRecs();
void DisposeOutRec(PolyOutList::size_type index);
void SwapPositionsInAEL(TEdge *edge1, TEdge *edge2);
void DeleteFromAEL(TEdge *e);
void UpdateEdgeIntoAEL(TEdge *&e);
typedef std::vector<LocalMinimum> MinimaList;
MinimaList::iterator m_CurrentLM;
MinimaList m_MinimaList;
bool m_UseFullRange;
EdgeList m_edges;
bool m_PreserveCollinear;
bool m_HasOpenPaths;
PolyOutList m_PolyOuts;
TEdge *m_ActiveEdges;
typedef std::priority_queue<cInt> ScanbeamList;
ScanbeamList m_Scanbeam;
};
//------------------------------------------------------------------------------
class Clipper : public virtual ClipperBase {
public:
Clipper(int initOptions = 0);
bool Execute(ClipType clipType, Paths &solution,
PolyFillType fillType = pftEvenOdd);
bool Execute(ClipType clipType, Paths &solution, PolyFillType subjFillType,
PolyFillType clipFillType);
bool Execute(ClipType clipType, PolyTree &polytree,
PolyFillType fillType = pftEvenOdd);
bool Execute(ClipType clipType, PolyTree &polytree, PolyFillType subjFillType,
PolyFillType clipFillType);
bool ReverseSolution() { return m_ReverseOutput; };
void ReverseSolution(bool value) { m_ReverseOutput = value; };
bool StrictlySimple() { return m_StrictSimple; };
void StrictlySimple(bool value) { m_StrictSimple = value; };
// set the callback function for z value filling on intersections (otherwise Z
// is 0)
#ifdef use_xyz
void ZFillFunction(ZFillCallback zFillFunc);
#endif
protected:
virtual bool ExecuteInternal();
private:
JoinList m_Joins;
JoinList m_GhostJoins;
IntersectList m_IntersectList;
ClipType m_ClipType;
typedef std::list<cInt> MaximaList;
MaximaList m_Maxima;
TEdge *m_SortedEdges;
bool m_ExecuteLocked;
PolyFillType m_ClipFillType;
PolyFillType m_SubjFillType;
bool m_ReverseOutput;
bool m_UsingPolyTree;
bool m_StrictSimple;
#ifdef use_xyz
ZFillCallback m_ZFill; // custom callback
#endif
void SetWindingCount(TEdge &edge);
bool IsEvenOddFillType(const TEdge &edge) const;
bool IsEvenOddAltFillType(const TEdge &edge) const;
void InsertLocalMinimaIntoAEL(const cInt botY);
void InsertEdgeIntoAEL(TEdge *edge, TEdge *startEdge);
void AddEdgeToSEL(TEdge *edge);
bool PopEdgeFromSEL(TEdge *&edge);
void CopyAELToSEL();
void DeleteFromSEL(TEdge *e);
void SwapPositionsInSEL(TEdge *edge1, TEdge *edge2);
bool IsContributing(const TEdge &edge) const;
bool IsTopHorz(const cInt XPos);
void DoMaxima(TEdge *e);
void ProcessHorizontals();
void ProcessHorizontal(TEdge *horzEdge);
void AddLocalMaxPoly(TEdge *e1, TEdge *e2, const IntPoint &pt);
OutPt *AddLocalMinPoly(TEdge *e1, TEdge *e2, const IntPoint &pt);
OutRec *GetOutRec(int idx);
void AppendPolygon(TEdge *e1, TEdge *e2);
void IntersectEdges(TEdge *e1, TEdge *e2, IntPoint &pt);
OutPt *AddOutPt(TEdge *e, const IntPoint &pt);
OutPt *GetLastOutPt(TEdge *e);
bool ProcessIntersections(const cInt topY);
void BuildIntersectList(const cInt topY);
void ProcessIntersectList();
void ProcessEdgesAtTopOfScanbeam(const cInt topY);
void BuildResult(Paths &polys);
void BuildResult2(PolyTree &polytree);
void SetHoleState(TEdge *e, OutRec *outrec);
void DisposeIntersectNodes();
bool FixupIntersectionOrder();
void FixupOutPolygon(OutRec &outrec);
void FixupOutPolyline(OutRec &outrec);
bool IsHole(TEdge *e);
bool FindOwnerFromSplitRecs(OutRec &outRec, OutRec *&currOrfl);
void FixHoleLinkage(OutRec &outrec);
void AddJoin(OutPt *op1, OutPt *op2, const IntPoint offPt);
void ClearJoins();
void ClearGhostJoins();
void AddGhostJoin(OutPt *op, const IntPoint offPt);
bool JoinPoints(Join *j, OutRec *outRec1, OutRec *outRec2);
void JoinCommonEdges();
void DoSimplePolygons();
void FixupFirstLefts1(OutRec *OldOutRec, OutRec *NewOutRec);
void FixupFirstLefts2(OutRec *InnerOutRec, OutRec *OuterOutRec);
void FixupFirstLefts3(OutRec *OldOutRec, OutRec *NewOutRec);
#ifdef use_xyz
void SetZ(IntPoint &pt, TEdge &e1, TEdge &e2);
#endif
};
//------------------------------------------------------------------------------
class ClipperOffset {
public:
ClipperOffset(double miterLimit = 2.0, double roundPrecision = 0.25);
~ClipperOffset();
void AddPath(const Path &path, JoinType joinType, EndType endType);
void AddPaths(const Paths &paths, JoinType joinType, EndType endType);
void Execute(Paths &solution, double delta);
void Execute(PolyTree &solution, double delta);
void Clear();
double MiterLimit;
double ArcTolerance;
private:
Paths m_destPolys;
Path m_srcPoly;
Path m_destPoly;
std::vector<DoublePoint> m_normals;
double m_delta, m_sinA, m_sin, m_cos;
double m_miterLim, m_StepsPerRad;
IntPoint m_lowest;
PolyNode m_polyNodes;
void FixOrientations();
void DoOffset(double delta);
void OffsetPoint(int j, int &k, JoinType jointype);
void DoSquare(int j, int k);
void DoMiter(int j, int k, double r);
void DoRound(int j, int k);
};
//------------------------------------------------------------------------------
class clipperException : public std::exception {
public:
clipperException(const char *description) : m_descr(description) {}
virtual ~clipperException() throw() {}
virtual const char *what() const throw() { return m_descr.c_str(); }
private:
std::string m_descr;
};
//------------------------------------------------------------------------------
} // ClipperLib namespace
#endif // clipper_hpp

View File

@@ -0,0 +1,46 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ocr_cls_process.h"
#include <cmath>
#include <cstring>
#include <fstream>
#include <iostream>
#include <iostream>
#include <vector>
const std::vector<int> CLS_IMAGE_SHAPE = {3, 48, 192};
cv::Mat cls_resize_img(const cv::Mat &img) {
int imgC = CLS_IMAGE_SHAPE[0];
int imgW = CLS_IMAGE_SHAPE[2];
int imgH = CLS_IMAGE_SHAPE[1];
float ratio = float(img.cols) / float(img.rows);
int resize_w = 0;
if (ceilf(imgH * ratio) > imgW)
resize_w = imgW;
else
resize_w = int(ceilf(imgH * ratio));
cv::Mat resize_img;
cv::resize(img, resize_img, cv::Size(resize_w, imgH), 0.f, 0.f,
cv::INTER_CUBIC);
if (resize_w < imgW) {
cv::copyMakeBorder(resize_img, resize_img, 0, 0, 0, int(imgW - resize_w),
cv::BORDER_CONSTANT, {0, 0, 0});
}
return resize_img;
}

View File

@@ -0,0 +1,23 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "common.h"
#include <opencv2/opencv.hpp>
#include <vector>
extern const std::vector<int> CLS_IMAGE_SHAPE;
cv::Mat cls_resize_img(const cv::Mat &img);

View File

@@ -0,0 +1,144 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ocr_crnn_process.h"
#include <cmath>
#include <cstring>
#include <fstream>
#include <iostream>
#include <iostream>
#include <vector>
const std::string CHARACTER_TYPE = "ch";
const int MAX_DICT_LENGTH = 6624;
const std::vector<int> REC_IMAGE_SHAPE = {3, 48, 320};
static cv::Mat crnn_resize_norm_img(cv::Mat img, float wh_ratio) {
int imgC = REC_IMAGE_SHAPE[0];
int imgH = REC_IMAGE_SHAPE[1];
int imgW = REC_IMAGE_SHAPE[2];
if (CHARACTER_TYPE == "ch")
imgW = int(32 * wh_ratio);
float ratio = float(img.cols) / float(img.rows);
int resize_w = 0;
if (ceilf(imgH * ratio) > imgW)
resize_w = imgW;
else
resize_w = int(ceilf(imgH * ratio));
cv::Mat resize_img;
cv::resize(img, resize_img, cv::Size(resize_w, imgH), 0.f, 0.f,
cv::INTER_CUBIC);
resize_img.convertTo(resize_img, CV_32FC3, 1 / 255.f);
for (int h = 0; h < resize_img.rows; h++) {
for (int w = 0; w < resize_img.cols; w++) {
resize_img.at<cv::Vec3f>(h, w)[0] =
(resize_img.at<cv::Vec3f>(h, w)[0] - 0.5) * 2;
resize_img.at<cv::Vec3f>(h, w)[1] =
(resize_img.at<cv::Vec3f>(h, w)[1] - 0.5) * 2;
resize_img.at<cv::Vec3f>(h, w)[2] =
(resize_img.at<cv::Vec3f>(h, w)[2] - 0.5) * 2;
}
}
cv::Mat dist;
cv::copyMakeBorder(resize_img, dist, 0, 0, 0, int(imgW - resize_w),
cv::BORDER_CONSTANT, {0, 0, 0});
return dist;
}
cv::Mat crnn_resize_img(const cv::Mat &img, float wh_ratio) {
int imgC = REC_IMAGE_SHAPE[0];
int imgH = REC_IMAGE_SHAPE[1];
int imgW = REC_IMAGE_SHAPE[2];
imgW = int(imgH * wh_ratio);
float ratio = float(img.cols) / float(img.rows);
int resize_w = 0;
if (ceilf(imgH * ratio) > imgW)
resize_w = imgW;
else
resize_w = int(ceilf(imgH * ratio));
cv::Mat resize_img;
cv::resize(img, resize_img, cv::Size(resize_w, imgH), 0.f, 0.f,
cv::INTER_LINEAR);
cv::copyMakeBorder(resize_img, resize_img, 0, 0, 0,
int(imgW - resize_img.cols), cv::BORDER_CONSTANT,
{127, 127, 127});
return resize_img;
}
cv::Mat get_rotate_crop_image(const cv::Mat &srcimage,
const std::vector<std::vector<int>> &box) {
std::vector<std::vector<int>> points = box;
int x_collect[4] = {box[0][0], box[1][0], box[2][0], box[3][0]};
int y_collect[4] = {box[0][1], box[1][1], box[2][1], box[3][1]};
int left = int(*std::min_element(x_collect, x_collect + 4));
int right = int(*std::max_element(x_collect, x_collect + 4));
int top = int(*std::min_element(y_collect, y_collect + 4));
int bottom = int(*std::max_element(y_collect, y_collect + 4));
cv::Mat img_crop;
srcimage(cv::Rect(left, top, right - left, bottom - top)).copyTo(img_crop);
for (int i = 0; i < points.size(); i++) {
points[i][0] -= left;
points[i][1] -= top;
}
int img_crop_width = int(sqrt(pow(points[0][0] - points[1][0], 2) +
pow(points[0][1] - points[1][1], 2)));
int img_crop_height = int(sqrt(pow(points[0][0] - points[3][0], 2) +
pow(points[0][1] - points[3][1], 2)));
cv::Point2f pts_std[4];
pts_std[0] = cv::Point2f(0., 0.);
pts_std[1] = cv::Point2f(img_crop_width, 0.);
pts_std[2] = cv::Point2f(img_crop_width, img_crop_height);
pts_std[3] = cv::Point2f(0.f, img_crop_height);
cv::Point2f pointsf[4];
pointsf[0] = cv::Point2f(points[0][0], points[0][1]);
pointsf[1] = cv::Point2f(points[1][0], points[1][1]);
pointsf[2] = cv::Point2f(points[2][0], points[2][1]);
pointsf[3] = cv::Point2f(points[3][0], points[3][1]);
cv::Mat M = cv::getPerspectiveTransform(pointsf, pts_std);
cv::Mat dst_img;
cv::warpPerspective(img_crop, dst_img, M,
cv::Size(img_crop_width, img_crop_height),
cv::BORDER_REPLICATE);
if (float(dst_img.rows) >= float(dst_img.cols) * 1.5) {
/*
cv::Mat srcCopy = cv::Mat(dst_img.rows, dst_img.cols, dst_img.depth());
cv::transpose(dst_img, srcCopy);
cv::flip(srcCopy, srcCopy, 0);
return srcCopy;
*/
cv::transpose(dst_img, dst_img);
cv::flip(dst_img, dst_img, 0);
return dst_img;
} else {
return dst_img;
}
}

View File

@@ -0,0 +1,20 @@
//
// Created by fujiayi on 2020/7/3.
//
#pragma once
#include "common.h"
#include <opencv2/opencv.hpp>
#include <vector>
extern const std::vector<int> REC_IMAGE_SHAPE;
cv::Mat get_rotate_crop_image(const cv::Mat &srcimage,
const std::vector<std::vector<int>> &box);
cv::Mat crnn_resize_img(const cv::Mat &img, float wh_ratio);
template <class ForwardIterator>
inline size_t argmax(ForwardIterator first, ForwardIterator last) {
return std::distance(first, std::max_element(first, last));
}

View File

@@ -0,0 +1,342 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ocr_clipper.hpp"
#include "opencv2/core.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>
#include <math.h>
#include <vector>
static void getcontourarea(float **box, float unclip_ratio, float &distance) {
int pts_num = 4;
float area = 0.0f;
float dist = 0.0f;
for (int i = 0; i < pts_num; i++) {
area += box[i][0] * box[(i + 1) % pts_num][1] -
box[i][1] * box[(i + 1) % pts_num][0];
dist += sqrtf((box[i][0] - box[(i + 1) % pts_num][0]) *
(box[i][0] - box[(i + 1) % pts_num][0]) +
(box[i][1] - box[(i + 1) % pts_num][1]) *
(box[i][1] - box[(i + 1) % pts_num][1]));
}
area = fabs(float(area / 2.0));
distance = area * unclip_ratio / dist;
}
static cv::RotatedRect unclip(float **box) {
float unclip_ratio = 2.0;
float distance = 1.0;
getcontourarea(box, unclip_ratio, distance);
ClipperLib::ClipperOffset offset;
ClipperLib::Path p;
p << ClipperLib::IntPoint(int(box[0][0]), int(box[0][1]))
<< ClipperLib::IntPoint(int(box[1][0]), int(box[1][1]))
<< ClipperLib::IntPoint(int(box[2][0]), int(box[2][1]))
<< ClipperLib::IntPoint(int(box[3][0]), int(box[3][1]));
offset.AddPath(p, ClipperLib::jtRound, ClipperLib::etClosedPolygon);
ClipperLib::Paths soln;
offset.Execute(soln, distance);
std::vector<cv::Point2f> points;
for (int j = 0; j < soln.size(); j++) {
for (int i = 0; i < soln[soln.size() - 1].size(); i++) {
points.emplace_back(soln[j][i].X, soln[j][i].Y);
}
}
cv::RotatedRect res = cv::minAreaRect(points);
return res;
}
static float **Mat2Vec(cv::Mat mat) {
auto **array = new float *[mat.rows];
for (int i = 0; i < mat.rows; ++i) {
array[i] = new float[mat.cols];
}
for (int i = 0; i < mat.rows; ++i) {
for (int j = 0; j < mat.cols; ++j) {
array[i][j] = mat.at<float>(i, j);
}
}
return array;
}
static void quickSort(float **s, int l, int r) {
if (l < r) {
int i = l, j = r;
float x = s[l][0];
float *xp = s[l];
while (i < j) {
while (i < j && s[j][0] >= x) {
j--;
}
if (i < j) {
std::swap(s[i++], s[j]);
}
while (i < j && s[i][0] < x) {
i++;
}
if (i < j) {
std::swap(s[j--], s[i]);
}
}
s[i] = xp;
quickSort(s, l, i - 1);
quickSort(s, i + 1, r);
}
}
static void quickSort_vector(std::vector<std::vector<int>> &box, int l, int r,
int axis) {
if (l < r) {
int i = l, j = r;
int x = box[l][axis];
std::vector<int> xp(box[l]);
while (i < j) {
while (i < j && box[j][axis] >= x) {
j--;
}
if (i < j) {
std::swap(box[i++], box[j]);
}
while (i < j && box[i][axis] < x) {
i++;
}
if (i < j) {
std::swap(box[j--], box[i]);
}
}
box[i] = xp;
quickSort_vector(box, l, i - 1, axis);
quickSort_vector(box, i + 1, r, axis);
}
}
static std::vector<std::vector<int>>
order_points_clockwise(std::vector<std::vector<int>> pts) {
std::vector<std::vector<int>> box = pts;
quickSort_vector(box, 0, int(box.size() - 1), 0);
std::vector<std::vector<int>> leftmost = {box[0], box[1]};
std::vector<std::vector<int>> rightmost = {box[2], box[3]};
if (leftmost[0][1] > leftmost[1][1]) {
std::swap(leftmost[0], leftmost[1]);
}
if (rightmost[0][1] > rightmost[1][1]) {
std::swap(rightmost[0], rightmost[1]);
}
std::vector<std::vector<int>> rect = {leftmost[0], rightmost[0], rightmost[1],
leftmost[1]};
return rect;
}
static float **get_mini_boxes(cv::RotatedRect box, float &ssid) {
ssid = box.size.width >= box.size.height ? box.size.height : box.size.width;
cv::Mat points;
cv::boxPoints(box, points);
// sorted box points
auto array = Mat2Vec(points);
quickSort(array, 0, 3);
float *idx1 = array[0], *idx2 = array[1], *idx3 = array[2], *idx4 = array[3];
if (array[3][1] <= array[2][1]) {
idx2 = array[3];
idx3 = array[2];
} else {
idx2 = array[2];
idx3 = array[3];
}
if (array[1][1] <= array[0][1]) {
idx1 = array[1];
idx4 = array[0];
} else {
idx1 = array[0];
idx4 = array[1];
}
array[0] = idx1;
array[1] = idx2;
array[2] = idx3;
array[3] = idx4;
return array;
}
template <class T> T clamp(T x, T min, T max) {
if (x > max) {
return max;
}
if (x < min) {
return min;
}
return x;
}
static float clampf(float x, float min, float max) {
if (x > max)
return max;
if (x < min)
return min;
return x;
}
float box_score_fast(float **box_array, cv::Mat pred) {
auto array = box_array;
int width = pred.cols;
int height = pred.rows;
float box_x[4] = {array[0][0], array[1][0], array[2][0], array[3][0]};
float box_y[4] = {array[0][1], array[1][1], array[2][1], array[3][1]};
int xmin = clamp(int(std::floorf(*(std::min_element(box_x, box_x + 4)))), 0,
width - 1);
int xmax = clamp(int(std::ceilf(*(std::max_element(box_x, box_x + 4)))), 0,
width - 1);
int ymin = clamp(int(std::floorf(*(std::min_element(box_y, box_y + 4)))), 0,
height - 1);
int ymax = clamp(int(std::ceilf(*(std::max_element(box_y, box_y + 4)))), 0,
height - 1);
cv::Mat mask;
mask = cv::Mat::zeros(ymax - ymin + 1, xmax - xmin + 1, CV_8UC1);
cv::Point root_point[4];
root_point[0] = cv::Point(int(array[0][0]) - xmin, int(array[0][1]) - ymin);
root_point[1] = cv::Point(int(array[1][0]) - xmin, int(array[1][1]) - ymin);
root_point[2] = cv::Point(int(array[2][0]) - xmin, int(array[2][1]) - ymin);
root_point[3] = cv::Point(int(array[3][0]) - xmin, int(array[3][1]) - ymin);
const cv::Point *ppt[1] = {root_point};
int npt[] = {4};
cv::fillPoly(mask, ppt, npt, 1, cv::Scalar(1));
cv::Mat croppedImg;
pred(cv::Rect(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1))
.copyTo(croppedImg);
auto score = cv::mean(croppedImg, mask)[0];
return score;
}
std::vector<std::vector<std::vector<int>>>
boxes_from_bitmap(const cv::Mat &pred, const cv::Mat &bitmap) {
const int min_size = 3;
const int max_candidates = 1000;
const float box_thresh = 0.5;
int width = bitmap.cols;
int height = bitmap.rows;
std::vector<std::vector<cv::Point>> contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(bitmap, contours, hierarchy, cv::RETR_LIST,
cv::CHAIN_APPROX_SIMPLE);
int num_contours =
contours.size() >= max_candidates ? max_candidates : contours.size();
std::vector<std::vector<std::vector<int>>> boxes;
for (int _i = 0; _i < num_contours; _i++) {
float ssid;
cv::RotatedRect box = cv::minAreaRect(contours[_i]);
auto array = get_mini_boxes(box, ssid);
auto box_for_unclip = array;
// end get_mini_box
if (ssid < min_size) {
continue;
}
float score;
score = box_score_fast(array, pred);
// end box_score_fast
if (score < box_thresh) {
continue;
}
// start for unclip
cv::RotatedRect points = unclip(box_for_unclip);
// end for unclip
cv::RotatedRect clipbox = points;
auto cliparray = get_mini_boxes(clipbox, ssid);
if (ssid < min_size + 2)
continue;
int dest_width = pred.cols;
int dest_height = pred.rows;
std::vector<std::vector<int>> intcliparray;
for (int num_pt = 0; num_pt < 4; num_pt++) {
std::vector<int> a{int(clampf(roundf(cliparray[num_pt][0] / float(width) *
float(dest_width)),
0, float(dest_width))),
int(clampf(roundf(cliparray[num_pt][1] /
float(height) * float(dest_height)),
0, float(dest_height)))};
intcliparray.emplace_back(std::move(a));
}
boxes.emplace_back(std::move(intcliparray));
} // end for
return boxes;
}
int _max(int a, int b) { return a >= b ? a : b; }
int _min(int a, int b) { return a >= b ? b : a; }
std::vector<std::vector<std::vector<int>>>
filter_tag_det_res(const std::vector<std::vector<std::vector<int>>> &o_boxes,
float ratio_h, float ratio_w, const cv::Mat &srcimg) {
int oriimg_h = srcimg.rows;
int oriimg_w = srcimg.cols;
std::vector<std::vector<std::vector<int>>> boxes{o_boxes};
std::vector<std::vector<std::vector<int>>> root_points;
for (int n = 0; n < boxes.size(); n++) {
boxes[n] = order_points_clockwise(boxes[n]);
for (int m = 0; m < boxes[0].size(); m++) {
boxes[n][m][0] /= ratio_w;
boxes[n][m][1] /= ratio_h;
boxes[n][m][0] = int(_min(_max(boxes[n][m][0], 0), oriimg_w - 1));
boxes[n][m][1] = int(_min(_max(boxes[n][m][1], 0), oriimg_h - 1));
}
}
for (int n = 0; n < boxes.size(); n++) {
int rect_width, rect_height;
rect_width = int(sqrt(pow(boxes[n][0][0] - boxes[n][1][0], 2) +
pow(boxes[n][0][1] - boxes[n][1][1], 2)));
rect_height = int(sqrt(pow(boxes[n][0][0] - boxes[n][3][0], 2) +
pow(boxes[n][0][1] - boxes[n][3][1], 2)));
if (rect_width <= 10 || rect_height <= 10)
continue;
root_points.push_back(boxes[n]);
}
return root_points;
}

View File

@@ -0,0 +1,13 @@
//
// Created by fujiayi on 2020/7/2.
//
#pragma once
#include <opencv2/opencv.hpp>
#include <vector>
std::vector<std::vector<std::vector<int>>>
boxes_from_bitmap(const cv::Mat &pred, const cv::Mat &bitmap);
std::vector<std::vector<std::vector<int>>>
filter_tag_det_res(const std::vector<std::vector<std::vector<int>>> &o_boxes,
float ratio_h, float ratio_w, const cv::Mat &srcimg);

View File

@@ -0,0 +1,350 @@
//
// Created by fujiayi on 2020/7/1.
//
#include "ocr_ppredictor.h"
#include "common.h"
#include "ocr_cls_process.h"
#include "ocr_crnn_process.h"
#include "ocr_db_post_process.h"
#include "preprocess.h"
namespace ppredictor {
OCR_PPredictor::OCR_PPredictor(const OCR_Config &config) : _config(config) {}
int OCR_PPredictor::init(const std::string &det_model_content,
const std::string &rec_model_content,
const std::string &cls_model_content) {
_det_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl,_config.thread_num, NET_OCR, _config.mode});
_det_predictor->init_nb(det_model_content);
_rec_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl,_config.thread_num, NET_OCR_INTERNAL, _config.mode});
_rec_predictor->init_nb(rec_model_content);
_cls_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl,_config.thread_num, NET_OCR_INTERNAL, _config.mode});
_cls_predictor->init_nb(cls_model_content);
return RETURN_OK;
}
int OCR_PPredictor::init_from_file(const std::string &det_model_path,
const std::string &rec_model_path,
const std::string &cls_model_path) {
_det_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl, _config.thread_num, NET_OCR, _config.mode});
_det_predictor->init_from_file(det_model_path);
_rec_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl,_config.thread_num, NET_OCR_INTERNAL, _config.mode});
_rec_predictor->init_from_file(rec_model_path);
_cls_predictor = std::unique_ptr<PPredictor>(
new PPredictor{_config.use_opencl,_config.thread_num, NET_OCR_INTERNAL, _config.mode});
_cls_predictor->init_from_file(cls_model_path);
return RETURN_OK;
}
/**
* for debug use, show result of First Step
* @param filter_boxes
* @param boxes
* @param srcimg
*/
static void
visual_img(const std::vector<std::vector<std::vector<int>>> &filter_boxes,
const std::vector<std::vector<std::vector<int>>> &boxes,
const cv::Mat &srcimg) {
// visualization
cv::Point rook_points[filter_boxes.size()][4];
for (int n = 0; n < filter_boxes.size(); n++) {
for (int m = 0; m < filter_boxes[0].size(); m++) {
rook_points[n][m] =
cv::Point(int(filter_boxes[n][m][0]), int(filter_boxes[n][m][1]));
}
}
cv::Mat img_vis;
srcimg.copyTo(img_vis);
for (int n = 0; n < boxes.size(); n++) {
const cv::Point *ppt[1] = {rook_points[n]};
int npt[] = {4};
cv::polylines(img_vis, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, 8, 0);
}
// 调试用,自行替换需要修改的路径
cv::imwrite("/sdcard/1/vis.png", img_vis);
}
std::vector<OCRPredictResult>
OCR_PPredictor::infer_ocr(cv::Mat &origin,int max_size_len, int run_det, int run_cls, int run_rec) {
LOGI("ocr cpp start *****************");
LOGI("ocr cpp det: %d, cls: %d, rec: %d", run_det, run_cls, run_rec);
std::vector<OCRPredictResult> ocr_results;
if (run_det || run_rec) {
infer_det(origin, max_size_len, ocr_results);
}
if (run_rec) {
if (ocr_results.empty()) {
OCRPredictResult res;
ocr_results.emplace_back(std::move(res));
}
for (auto & ocr_result : ocr_results) {
infer_rec(origin, run_cls, ocr_result);
}
}
if (run_cls) {
ClsPredictResult cls_res = infer_cls(origin);
OCRPredictResult res;
res.cls_score = cls_res.cls_score;
res.cls_label = cls_res.cls_label;
ocr_results.push_back(res);
}
LOGI("ocr cpp end *****************");
return ocr_results;
}
cv::Mat DetResizeImg(const cv::Mat img, int max_size_len,
std::vector<float> &ratio_hw) {
int w = img.cols;
int h = img.rows;
float ratio = 1.f;
int max_wh = w >= h ? w : h;
if (max_wh > max_size_len) {
if (h > w) {
ratio = static_cast<float>(max_size_len) / static_cast<float>(h);
} else {
ratio = static_cast<float>(max_size_len) / static_cast<float>(w);
}
}
int resize_h = static_cast<int>(float(h) * ratio);
int resize_w = static_cast<int>(float(w) * ratio);
int shapeSize = 32;
if (resize_h % shapeSize == 0)
resize_h = resize_h;
else if (resize_h / shapeSize < 1 + 1e-5)
resize_h = shapeSize;
else
resize_h = (resize_h / shapeSize - 1) * shapeSize;
if (resize_w % shapeSize == 0)
resize_w = resize_w;
else if (resize_w / shapeSize < 1 + 1e-5)
resize_w = shapeSize;
else
resize_w = (resize_w / shapeSize - 1) * shapeSize;
cv::Mat resize_img;
cv::resize(img, resize_img, cv::Size(resize_w, resize_h));
ratio_hw.push_back(static_cast<float>(resize_h) / static_cast<float>(h));
ratio_hw.push_back(static_cast<float>(resize_w) / static_cast<float>(w));
return resize_img;
}
void OCR_PPredictor::infer_det(cv::Mat &origin, int max_size_len, std::vector<OCRPredictResult> &ocr_results) {
std::vector<float> mean = {0.485f, 0.456f, 0.406f};
std::vector<float> scale = {1 / 0.229f, 1 / 0.224f, 1 / 0.225f};
PredictorInput input = _det_predictor->get_first_input();
std::vector<float> ratio_hw;
cv::Mat input_image = DetResizeImg(origin, max_size_len, ratio_hw);
input_image.convertTo(input_image, CV_32FC3, 1 / 255.0f);
const float *dimg = reinterpret_cast<const float *>(input_image.data);
int input_size = input_image.rows * input_image.cols;
input.set_dims({1, 3, input_image.rows, input_image.cols});
neon_mean_scale(dimg, input.get_mutable_float_data(), input_size, mean,
scale);
LOGI("ocr cpp det shape %d,%d", input_image.rows,input_image.cols);
std::vector<PredictorOutput> results = _det_predictor->infer();
PredictorOutput &res = results.at(0);
std::vector<std::vector<std::vector<int>>> filtered_box = calc_filtered_boxes(
res.get_float_data(), res.get_size(), input_image.rows, input_image.cols, origin);
LOGI("ocr cpp det Filter_box size %ld", filtered_box.size());
for(auto & i : filtered_box){
LOGI("ocr cpp box %d,%d,%d,%d,%d,%d,%d,%d", i[0][0],i[0][1], i[1][0],i[1][1], i[2][0],i[2][1], i[3][0],i[3][1]);
OCRPredictResult res;
res.points = i;
ocr_results.push_back(res);
}
}
void OCR_PPredictor::infer_rec(const cv::Mat &origin_img, int run_cls, OCRPredictResult& ocr_result) {
std::vector<float> mean = {0.5f, 0.5f, 0.5f};
std::vector<float> scale = {1 / 0.5f, 1 / 0.5f, 1 / 0.5f};
std::vector<int64_t> dims = {1, 3, 0, 0};
PredictorInput input = _rec_predictor->get_first_input();
const std::vector<std::vector<int>> &box = ocr_result.points;
cv::Mat crop_img;
if (!box.empty()) {
crop_img = get_rotate_crop_image(origin_img, box);
} else {
crop_img = origin_img;
}
if (run_cls) {
ClsPredictResult cls_res = infer_cls(crop_img);
crop_img = cls_res.img;
ocr_result.cls_score = cls_res.cls_score;
ocr_result.cls_label = cls_res.cls_label;
}
float wh_ratio = float(crop_img.cols) / float(crop_img.rows);
cv::Mat input_image = crnn_resize_img(crop_img, wh_ratio);
input_image.convertTo(input_image, CV_32FC3, 1 / 255.0f);
const float *dimg = reinterpret_cast<const float *>(input_image.data);
int input_size = input_image.rows * input_image.cols;
dims[2] = input_image.rows;
dims[3] = input_image.cols;
input.set_dims(dims);
neon_mean_scale(dimg, input.get_mutable_float_data(), input_size, mean,
scale);
std::vector<PredictorOutput> results = _rec_predictor->infer();
const float *predict_batch = results.at(0).get_float_data();
const std::vector<int64_t> predict_shape = results.at(0).get_shape();
// ctc decode
int argmax_idx;
int last_index = 0;
float score = 0.f;
int count = 0;
float max_value = 0.0f;
for (int n = 0; n < predict_shape[1]; n++) {
argmax_idx = int(argmax(&predict_batch[n * predict_shape[2]],
&predict_batch[(n + 1) * predict_shape[2]]));
max_value =
float(*std::max_element(&predict_batch[n * predict_shape[2]],
&predict_batch[(n + 1) * predict_shape[2]]));
if (argmax_idx > 0 && (!(n > 0 && argmax_idx == last_index))) {
score += max_value;
count += 1;
ocr_result.word_index.push_back(argmax_idx);
}
last_index = argmax_idx;
}
score /= count;
ocr_result.score = score;
LOGI("ocr cpp rec word size %ld", count);
}
ClsPredictResult OCR_PPredictor::infer_cls(const cv::Mat &img, float thresh) {
std::vector<float> mean = {0.5f, 0.5f, 0.5f};
std::vector<float> scale = {1 / 0.5f, 1 / 0.5f, 1 / 0.5f};
std::vector<int64_t> dims = {1, 3, 0, 0};
PredictorInput input = _cls_predictor->get_first_input();
cv::Mat input_image = cls_resize_img(img);
input_image.convertTo(input_image, CV_32FC3, 1 / 255.0f);
const float *dimg = reinterpret_cast<const float *>(input_image.data);
int input_size = input_image.rows * input_image.cols;
dims[2] = input_image.rows;
dims[3] = input_image.cols;
input.set_dims(dims);
neon_mean_scale(dimg, input.get_mutable_float_data(), input_size, mean,
scale);
std::vector<PredictorOutput> results = _cls_predictor->infer();
const float *scores = results.at(0).get_float_data();
float score = 0;
int label = 0;
for (int64_t i = 0; i < results.at(0).get_size(); i++) {
LOGI("ocr cpp cls output scores [%f]", scores[i]);
if (scores[i] > score) {
score = scores[i];
label = i;
}
}
cv::Mat srcimg;
img.copyTo(srcimg);
if (label % 2 == 1 && score > thresh) {
cv::rotate(srcimg, srcimg, 1);
}
ClsPredictResult res;
res.cls_label = label;
res.cls_score = score;
res.img = srcimg;
LOGI("ocr cpp cls word cls %ld, %f", label, score);
return res;
}
std::vector<std::vector<std::vector<int>>>
OCR_PPredictor::calc_filtered_boxes(const float *pred, int pred_size,
int output_height, int output_width,
const cv::Mat &origin) {
const double threshold = 0.3;
const double maxvalue = 1;
cv::Mat pred_map = cv::Mat::zeros(output_height, output_width, CV_32F);
memcpy(pred_map.data, pred, pred_size * sizeof(float));
cv::Mat cbuf_map;
pred_map.convertTo(cbuf_map, CV_8UC1);
cv::Mat bit_map;
cv::threshold(cbuf_map, bit_map, threshold, maxvalue, cv::THRESH_BINARY);
std::vector<std::vector<std::vector<int>>> boxes =
boxes_from_bitmap(pred_map, bit_map);
float ratio_h = output_height * 1.0f / origin.rows;
float ratio_w = output_width * 1.0f / origin.cols;
std::vector<std::vector<std::vector<int>>> filter_boxes =
filter_tag_det_res(boxes, ratio_h, ratio_w, origin);
return filter_boxes;
}
std::vector<int>
OCR_PPredictor::postprocess_rec_word_index(const PredictorOutput &res) {
const int *rec_idx = res.get_int_data();
const std::vector<std::vector<uint64_t>> rec_idx_lod = res.get_lod();
std::vector<int> pred_idx;
for (int n = int(rec_idx_lod[0][0]); n < int(rec_idx_lod[0][1] * 2); n += 2) {
pred_idx.emplace_back(rec_idx[n]);
}
return pred_idx;
}
float OCR_PPredictor::postprocess_rec_score(const PredictorOutput &res) {
const float *predict_batch = res.get_float_data();
const std::vector<int64_t> predict_shape = res.get_shape();
const std::vector<std::vector<uint64_t>> predict_lod = res.get_lod();
int blank = predict_shape[1];
float score = 0.f;
int count = 0;
for (int n = predict_lod[0][0]; n < predict_lod[0][1] - 1; n++) {
int argmax_idx = argmax(predict_batch + n * predict_shape[1],
predict_batch + (n + 1) * predict_shape[1]);
float max_value = predict_batch[n * predict_shape[1] + argmax_idx];
if (blank - 1 - argmax_idx > 1e-5) {
score += max_value;
count += 1;
}
}
if (count == 0) {
LOGE("calc score count 0");
} else {
score /= count;
}
LOGI("calc score: %f", score);
return score;
}
NET_TYPE OCR_PPredictor::get_net_flag() const { return NET_OCR; }
}

View File

@@ -0,0 +1,130 @@
//
// Created by fujiayi on 2020/7/1.
//
#pragma once
#include "ppredictor.h"
#include <opencv2/opencv.hpp>
#include <paddle_api.h>
#include <string>
namespace ppredictor {
/**
* Config
*/
struct OCR_Config {
int use_opencl = 0;
int thread_num = 4; // Thread num
paddle::lite_api::PowerMode mode =
paddle::lite_api::LITE_POWER_HIGH; // PaddleLite Mode
};
/**
* PolyGone Result
*/
struct OCRPredictResult {
std::vector<int> word_index;
std::vector<std::vector<int>> points;
float score;
float cls_score;
int cls_label=-1;
};
struct ClsPredictResult {
float cls_score;
int cls_label=-1;
cv::Mat img;
};
/**
* OCR there are 2 models
* 1. First model(det),select polygones to show where are the texts
* 2. crop from the origin images, use these polygones to infer
*/
class OCR_PPredictor : public PPredictor_Interface {
public:
OCR_PPredictor(const OCR_Config &config);
virtual ~OCR_PPredictor() {}
/**
* 初始化二个模型的Predictor
* @param det_model_content
* @param rec_model_content
* @return
*/
int init(const std::string &det_model_content,
const std::string &rec_model_content,
const std::string &cls_model_content);
int init_from_file(const std::string &det_model_path,
const std::string &rec_model_path,
const std::string &cls_model_path);
/**
* Return OCR result
* @param dims
* @param input_data
* @param input_len
* @param net_flag
* @param origin
* @return
*/
virtual std::vector<OCRPredictResult>
infer_ocr(cv::Mat &origin, int max_size_len, int run_det, int run_cls, int run_rec);
virtual NET_TYPE get_net_flag() const;
private:
/**
* calcul Polygone from the result image of first model
* @param pred
* @param output_height
* @param output_width
* @param origin
* @return
*/
std::vector<std::vector<std::vector<int>>>
calc_filtered_boxes(const float *pred, int pred_size, int output_height,
int output_width, const cv::Mat &origin);
void
infer_det(cv::Mat &origin, int max_side_len, std::vector<OCRPredictResult>& ocr_results);
/**
* infer for rec model
*
* @param boxes
* @param origin
* @return
*/
void
infer_rec(const cv::Mat &origin, int run_cls, OCRPredictResult& ocr_result);
/**
* infer for cls model
*
* @param boxes
* @param origin
* @return
*/
ClsPredictResult infer_cls(const cv::Mat &origin, float thresh = 0.9);
/**
* Postprocess or sencod model to extract text
* @param res
* @return
*/
std::vector<int> postprocess_rec_word_index(const PredictorOutput &res);
/**
* calculate confidence of second model text result
* @param res
* @return
*/
float postprocess_rec_score(const PredictorOutput &res);
std::unique_ptr<PPredictor> _det_predictor;
std::unique_ptr<PPredictor> _rec_predictor;
std::unique_ptr<PPredictor> _cls_predictor;
OCR_Config _config;
};
}

View File

@@ -0,0 +1,95 @@
#include "ppredictor.h"
#include "common.h"
namespace ppredictor {
PPredictor::PPredictor(int use_opencl, int thread_num, int net_flag,
paddle::lite_api::PowerMode mode)
: _use_opencl(use_opencl), _thread_num(thread_num), _net_flag(net_flag), _mode(mode) {}
int PPredictor::init_nb(const std::string &model_content) {
paddle::lite_api::MobileConfig config;
config.set_model_from_buffer(model_content);
return _init(config);
}
int PPredictor::init_from_file(const std::string &model_content) {
paddle::lite_api::MobileConfig config;
config.set_model_from_file(model_content);
return _init(config);
}
template <typename ConfigT> int PPredictor::_init(ConfigT &config) {
bool is_opencl_backend_valid = paddle::lite_api::IsOpenCLBackendValid(/*check_fp16_valid = false*/);
if (is_opencl_backend_valid) {
if (_use_opencl != 0) {
// Make sure you have write permission of the binary path.
// We strongly recommend each model has a unique binary name.
const std::string bin_path = "/data/local/tmp/";
const std::string bin_name = "lite_opencl_kernel.bin";
config.set_opencl_binary_path_name(bin_path, bin_name);
// opencl tune option
// CL_TUNE_NONE: 0
// CL_TUNE_RAPID: 1
// CL_TUNE_NORMAL: 2
// CL_TUNE_EXHAUSTIVE: 3
const std::string tuned_path = "/data/local/tmp/";
const std::string tuned_name = "lite_opencl_tuned.bin";
config.set_opencl_tune(paddle::lite_api::CL_TUNE_NORMAL, tuned_path, tuned_name);
// opencl precision option
// CL_PRECISION_AUTO: 0, first fp16 if valid, default
// CL_PRECISION_FP32: 1, force fp32
// CL_PRECISION_FP16: 2, force fp16
config.set_opencl_precision(paddle::lite_api::CL_PRECISION_FP32);
LOGI("ocr cpp device: running on gpu.");
}
} else {
LOGI("ocr cpp device: running on cpu.");
// you can give backup cpu nb model instead
// config.set_model_from_file(cpu_nb_model_dir);
}
config.set_threads(_thread_num);
config.set_power_mode(_mode);
_predictor = paddle::lite_api::CreatePaddlePredictor(config);
LOGI("ocr cpp paddle instance created");
return RETURN_OK;
}
PredictorInput PPredictor::get_input(int index) {
PredictorInput input{_predictor->GetInput(index), index, _net_flag};
_is_input_get = true;
return input;
}
std::vector<PredictorInput> PPredictor::get_inputs(int num) {
std::vector<PredictorInput> results;
for (int i = 0; i < num; i++) {
results.emplace_back(get_input(i));
}
return results;
}
PredictorInput PPredictor::get_first_input() { return get_input(0); }
std::vector<PredictorOutput> PPredictor::infer() {
LOGI("ocr cpp infer Run start %d", _net_flag);
std::vector<PredictorOutput> results;
if (!_is_input_get) {
return results;
}
_predictor->Run();
LOGI("ocr cpp infer Run end");
for (int i = 0; i < _predictor->GetOutputNames().size(); i++) {
std::unique_ptr<const paddle::lite_api::Tensor> output_tensor =
_predictor->GetOutput(i);
LOGI("ocr cpp output tensor[%d] size %ld", i, product(output_tensor->shape()));
PredictorOutput result{std::move(output_tensor), i, _net_flag};
results.emplace_back(std::move(result));
}
return results;
}
NET_TYPE PPredictor::get_net_flag() const { return (NET_TYPE)_net_flag; }
}

View File

@@ -0,0 +1,64 @@
#pragma once
#include "paddle_api.h"
#include "predictor_input.h"
#include "predictor_output.h"
namespace ppredictor {
/**
* PaddleLite Preditor Common Interface
*/
class PPredictor_Interface {
public:
virtual ~PPredictor_Interface() {}
virtual NET_TYPE get_net_flag() const = 0;
};
/**
* Common Predictor
*/
class PPredictor : public PPredictor_Interface {
public:
PPredictor(
int use_opencl, int thread_num, int net_flag = 0,
paddle::lite_api::PowerMode mode = paddle::lite_api::LITE_POWER_HIGH);
virtual ~PPredictor() {}
/**
* init paddlitelite opt model,nb format ,or use ini_paddle
* @param model_content
* @return 0
*/
virtual int init_nb(const std::string &model_content);
virtual int init_from_file(const std::string &model_content);
std::vector<PredictorOutput> infer();
std::shared_ptr<paddle::lite_api::PaddlePredictor> get_predictor() {
return _predictor;
}
virtual std::vector<PredictorInput> get_inputs(int num);
virtual PredictorInput get_input(int index);
virtual PredictorInput get_first_input();
virtual NET_TYPE get_net_flag() const;
protected:
template <typename ConfigT> int _init(ConfigT &config);
private:
int _use_opencl;
int _thread_num;
paddle::lite_api::PowerMode _mode;
std::shared_ptr<paddle::lite_api::PaddlePredictor> _predictor;
bool _is_input_get = false;
int _net_flag;
};
}

View File

@@ -0,0 +1,28 @@
#include "predictor_input.h"
namespace ppredictor {
void PredictorInput::set_dims(std::vector<int64_t> dims) {
// yolov3
if (_net_flag == 101 && _index == 1) {
_tensor->Resize({1, 2});
_tensor->mutable_data<int>()[0] = (int)dims.at(2);
_tensor->mutable_data<int>()[1] = (int)dims.at(3);
} else {
_tensor->Resize(dims);
}
_is_dims_set = true;
}
float *PredictorInput::get_mutable_float_data() {
if (!_is_dims_set) {
LOGE("PredictorInput::set_dims is not called");
}
return _tensor->mutable_data<float>();
}
void PredictorInput::set_data(const float *input_data, int input_float_len) {
float *input_raw_data = get_mutable_float_data();
memcpy(input_raw_data, input_data, input_float_len * sizeof(float));
}
}

View File

@@ -0,0 +1,26 @@
#pragma once
#include "common.h"
#include <paddle_api.h>
#include <vector>
namespace ppredictor {
class PredictorInput {
public:
PredictorInput(std::unique_ptr<paddle::lite_api::Tensor> &&tensor, int index,
int net_flag)
: _tensor(std::move(tensor)), _index(index), _net_flag(net_flag) {}
void set_dims(std::vector<int64_t> dims);
float *get_mutable_float_data();
void set_data(const float *input_data, int input_float_len);
private:
std::unique_ptr<paddle::lite_api::Tensor> _tensor;
bool _is_dims_set = false;
int _index;
int _net_flag;
};
}

View File

@@ -0,0 +1,26 @@
#include "predictor_output.h"
namespace ppredictor {
const float *PredictorOutput::get_float_data() const {
return _tensor->data<float>();
}
const int *PredictorOutput::get_int_data() const {
return _tensor->data<int>();
}
const std::vector<std::vector<uint64_t>> PredictorOutput::get_lod() const {
return _tensor->lod();
}
int64_t PredictorOutput::get_size() const {
if (_net_flag == NET_OCR) {
return _tensor->shape().at(2) * _tensor->shape().at(3);
} else {
return product(_tensor->shape());
}
}
const std::vector<int64_t> PredictorOutput::get_shape() const {
return _tensor->shape();
}
}

View File

@@ -0,0 +1,31 @@
#pragma once
#include "common.h"
#include <paddle_api.h>
#include <vector>
namespace ppredictor {
class PredictorOutput {
public:
PredictorOutput() {}
PredictorOutput(std::unique_ptr<const paddle::lite_api::Tensor> &&tensor,
int index, int net_flag)
: _tensor(std::move(tensor)), _index(index), _net_flag(net_flag) {}
const float *get_float_data() const;
const int *get_int_data() const;
int64_t get_size() const;
const std::vector<std::vector<uint64_t>> get_lod() const;
const std::vector<int64_t> get_shape() const;
std::vector<float> data; // return float, or use data_int
std::vector<int> data_int; // several layers return int ,or use data
std::vector<int64_t> shape; // PaddleLite output shape
std::vector<std::vector<uint64_t>> lod; // PaddleLite output lod
private:
std::unique_ptr<const paddle::lite_api::Tensor> _tensor;
int _index;
int _net_flag;
};
}

View File

@@ -0,0 +1,82 @@
#include "preprocess.h"
#include <android/bitmap.h>
cv::Mat bitmap_to_cv_mat(JNIEnv *env, jobject bitmap) {
AndroidBitmapInfo info;
int result = AndroidBitmap_getInfo(env, bitmap, &info);
if (result != ANDROID_BITMAP_RESULT_SUCCESS) {
LOGE("AndroidBitmap_getInfo failed, result: %d", result);
return cv::Mat{};
}
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
LOGE("Bitmap format is not RGBA_8888 !");
return cv::Mat{};
}
unsigned char *srcData = NULL;
AndroidBitmap_lockPixels(env, bitmap, (void **)&srcData);
cv::Mat mat = cv::Mat::zeros(info.height, info.width, CV_8UC4);
memcpy(mat.data, srcData, info.height * info.width * 4);
AndroidBitmap_unlockPixels(env, bitmap);
cv::cvtColor(mat, mat, cv::COLOR_RGBA2BGR);
/**
if (!cv::imwrite("/sdcard/1/copy.jpg", mat)){
LOGE("Write image failed " );
}
*/
return mat;
}
cv::Mat resize_img(const cv::Mat &img, int height, int width) {
if (img.rows == height && img.cols == width) {
return img;
}
cv::Mat new_img;
cv::resize(img, new_img, cv::Size(height, width));
return new_img;
}
// fill tensor with mean and scale and trans layout: nhwc -> nchw, neon speed up
void neon_mean_scale(const float *din, float *dout, int size,
const std::vector<float> &mean,
const std::vector<float> &scale) {
if (mean.size() != 3 || scale.size() != 3) {
LOGE("[ERROR] mean or scale size must equal to 3");
return;
}
float32x4_t vmean0 = vdupq_n_f32(mean[0]);
float32x4_t vmean1 = vdupq_n_f32(mean[1]);
float32x4_t vmean2 = vdupq_n_f32(mean[2]);
float32x4_t vscale0 = vdupq_n_f32(scale[0]);
float32x4_t vscale1 = vdupq_n_f32(scale[1]);
float32x4_t vscale2 = vdupq_n_f32(scale[2]);
float *dout_c0 = dout;
float *dout_c1 = dout + size;
float *dout_c2 = dout + size * 2;
int i = 0;
for (; i < size - 3; i += 4) {
float32x4x3_t vin3 = vld3q_f32(din);
float32x4_t vsub0 = vsubq_f32(vin3.val[0], vmean0);
float32x4_t vsub1 = vsubq_f32(vin3.val[1], vmean1);
float32x4_t vsub2 = vsubq_f32(vin3.val[2], vmean2);
float32x4_t vs0 = vmulq_f32(vsub0, vscale0);
float32x4_t vs1 = vmulq_f32(vsub1, vscale1);
float32x4_t vs2 = vmulq_f32(vsub2, vscale2);
vst1q_f32(dout_c0, vs0);
vst1q_f32(dout_c1, vs1);
vst1q_f32(dout_c2, vs2);
din += 12;
dout_c0 += 4;
dout_c1 += 4;
dout_c2 += 4;
}
for (; i < size; i++) {
*(dout_c0++) = (*(din++) - mean[0]) * scale[0];
*(dout_c1++) = (*(din++) - mean[1]) * scale[1];
*(dout_c2++) = (*(din++) - mean[2]) * scale[2];
}
}

View File

@@ -0,0 +1,12 @@
#pragma once
#include "common.h"
#include <jni.h>
#include <opencv2/opencv.hpp>
cv::Mat bitmap_to_cv_mat(JNIEnv *env, jobject bitmap);
cv::Mat resize_img(const cv::Mat &img, int height, int width);
void neon_mean_scale(const float *din, float *dout, int size,
const std::vector<float> &mean,
const std::vector<float> &scale);

View File

@@ -0,0 +1,127 @@
package com.baidu.paddle.lite.ocr;
import android.graphics.Bitmap;
import android.util.Log;
import org.opencv.android.OpenCVLoader;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
/**
* @author PaddleOCR
* Modified by TonyJiangWJ
* @since 2023-08-06
*/
public class OCRPredictorNative {
private static final AtomicBoolean isSOLoaded = new AtomicBoolean();
private static final ReentrantLock lock = new ReentrantLock();
public static void loadLibrary() throws RuntimeException {
if (!isSOLoaded.get() && isSOLoaded.compareAndSet(false, true)) {
try {
// 可能和AJ中的OpenCV冲突,直接初始化一遍
OpenCVLoader.initDebug();
System.loadLibrary("Native");
} catch (Throwable e) {
throw new RuntimeException(
"Load libNative.so failed, please check it exists in apk file.", e);
}
}
}
private Config config;
private long nativePointer = 0;
public OCRPredictorNative(Config config) {
lock.lock();
try {
this.config = config;
loadLibrary();
nativePointer = init(config.detModelFilename, config.recModelFilename, config.clsModelFilename, config.useOpencl,
config.cpuThreadNum, config.cpuPower);
Log.i("OCRPredictorNative", "load success " + nativePointer);
} finally {
lock.unlock();
}
}
public ArrayList<OcrResultModel> runImage(Bitmap originalImage, int max_size_len, int run_det, int run_cls, int run_rec) {
lock.lock();
try {
Log.i("OCRPredictorNative", "begin to run image");
float[] rawResults = forward(nativePointer, originalImage, max_size_len, run_det, run_cls, run_rec);
return postprocess(rawResults);
} finally {
lock.unlock();
}
}
public static class Config {
public int useOpencl;
public int cpuThreadNum;
public String cpuPower;
public String detModelFilename;
public String recModelFilename;
public String clsModelFilename;
}
public void destroy() {
if (nativePointer != 0) {
release(nativePointer);
nativePointer = 0;
}
}
protected native long init(String detModelPath, String recModelPath, String clsModelPath, int useOpencl, int threadNum, String cpuMode);
protected native float[] forward(long pointer, Bitmap originalImage, int max_size_len, int run_det, int run_cls, int run_rec);
protected native void release(long pointer);
private ArrayList<OcrResultModel> postprocess(float[] raw) {
ArrayList<OcrResultModel> results = new ArrayList<OcrResultModel>();
int begin = 0;
while (begin < raw.length) {
int point_num = Math.round(raw[begin]);
int word_num = Math.round(raw[begin + 1]);
OcrResultModel model = parse(raw, begin + 2, point_num, word_num);
begin += 2 + 1 + point_num * 2 + word_num + 2;
results.add(model);
}
return results;
}
private OcrResultModel parse(float[] raw, int begin, int pointNum, int wordNum) {
int current = begin;
OcrResultModel model = new OcrResultModel();
model.setConfidence(raw[current]);
current++;
for (int i = 0; i < pointNum; i++) {
model.addPoints(Math.round(raw[current + i * 2]), Math.round(raw[current + i * 2 + 1]));
}
current += (pointNum * 2);
for (int i = 0; i < wordNum; i++) {
int index = Math.round(raw[current + i]);
model.addWordIndex(index);
}
current += wordNum;
model.setClsIdx(raw[current]);
model.setClsConfidence(raw[current + 1]);
Log.i("OCRPredictorNative", "word finished " + wordNum);
return model;
}
@Override
protected void finalize() throws Throwable {
super.finalize();
destroy();
}
}

View File

@@ -0,0 +1,134 @@
package com.baidu.paddle.lite.ocr;
import android.graphics.Point;
import android.graphics.Rect;
import java.util.ArrayList;
import java.util.List;
/**
* @author PaddleOCR
* Modified by TonyJiangWJ
* @since 2023-08-06
*/
public class OcrResult implements Comparable<OcrResult> {
private String label;
private float confidence;
private Rect bounds;
private final List<OcrResult> elements = new ArrayList<>();
public OcrResult() {
}
public OcrResult(OcrResultModel resultModel) {
this.label = resultModel.getLabel();
this.confidence = resultModel.getConfidence();
int left = -1, right = -1, top = -1, bottom = -1;
for (Point point : resultModel.getPoints()) {
if (point.x < left || left == -1) {
left = point.x;
}
if (point.x > right || right == -1) {
right = point.x;
}
if (point.y < top || top == -1) {
top = point.y;
}
if (point.y > bottom || bottom == -1) {
bottom = point.y;
}
}
this.bounds = new Rect(left, top, right, bottom);
}
public OcrResult(String label, float confidence, Rect bounds) {
this.label = label;
this.confidence = confidence;
this.bounds = bounds;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public float getConfidence() {
return confidence;
}
public void setConfidence(float confidence) {
this.confidence = confidence;
}
public Rect getBounds() {
return bounds;
}
public void setBounds(Rect bounds) {
this.bounds = bounds;
}
public RectLocation getLocation() {
return new RectLocation(bounds);
}
public String getWords() {
return label.trim().replace("\r", "");
}
public List<OcrResult> getElements() {
return this.elements;
}
public void addElements(OcrResult element) {
this.elements.add(element);
}
@Override
public int compareTo(OcrResult o) {
// 上下差距小于二分之一的高度 判定为同一行
int deviation = Math.max(this.bounds.height(), o.bounds.height()) / 2;
// 通过垂直中心点的距离判定
if (Math.abs((this.bounds.top + this.bounds.bottom) / 2 - (o.bounds.top + o.bounds.bottom) / 2) < deviation) {
return this.bounds.left - o.bounds.left;
} else {
return this.bounds.bottom - o.bounds.bottom;
}
}
@Override
public String toString() {
return "OcrResult{" + "label='" + label + '\'' +
", confidence=" + confidence +
", bounds=" + bounds +
", elements=" + elements +
'}';
}
public static class RectLocation {
public int left;
public int top;
public int width;
public int height;
public RectLocation() {
}
public RectLocation(int left, int top, int width, int height) {
this.left = left;
this.top = top;
this.width = width;
this.height = height;
}
public RectLocation(Rect rect) {
left = rect.left;
top = rect.top;
width = rect.right - rect.left;
height = rect.bottom - rect.top;
}
}
}

View File

@@ -0,0 +1,97 @@
package com.baidu.paddle.lite.ocr;
import android.graphics.Point;
import java.util.ArrayList;
import java.util.List;
/**
* @author PaddleOCR
* Modified by TonyJiangWJ
* @since 2023-08-06
*/
public class OcrResultModel {
private final List<Point> points;
private final List<Integer> wordIndex;
private String label;
private float confidence;
private float clsIdx;
private String clsLabel;
private float clsConfidence;
public OcrResultModel() {
super();
points = new ArrayList<>();
wordIndex = new ArrayList<>();
}
public void addPoints(int x, int y) {
Point point = new Point(x, y);
points.add(point);
}
public void addWordIndex(int index) {
wordIndex.add(index);
}
public List<Point> getPoints() {
return points;
}
public List<Integer> getWordIndex() {
return wordIndex;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public float getConfidence() {
return confidence;
}
public void setConfidence(float confidence) {
this.confidence = confidence;
}
public float getClsIdx() {
return clsIdx;
}
public void setClsIdx(float idx) {
this.clsIdx = idx;
}
public String getClsLabel() {
return clsLabel;
}
public void setClsLabel(String label) {
this.clsLabel = label;
}
public float getClsConfidence() {
return clsConfidence;
}
public void setClsConfidence(float confidence) {
this.clsConfidence = confidence;
}
@Override
public String toString() {
return "OcrResultModel{" +
"points=" + points +
", wordIndex=" + wordIndex +
", label='" + label + '\'' +
", confidence=" + confidence +
", clsIdx=" + clsIdx +
", clsLabel='" + clsLabel + '\'' +
", clsConfidence=" + clsConfidence +
'}';
}
}

View File

@@ -0,0 +1,338 @@
package com.baidu.paddle.lite.ocr;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.preference.PreferenceManager;
import android.util.Base64;
import android.util.Log;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Vector;
/**
* @author PaddleOCR
* Modified by TonyJiangWJ
* @since 2023-08-06
*/
public class Predictor {
private static final String TAG = Predictor.class.getSimpleName();
public boolean isLoaded = false;
public int warmupIterNum = 1;
public int inferIterNum = 1;
public int cpuThreadNum = 4;
public String cpuPowerMode = "LITE_POWER_HIGH";
public String modelPath = "";
public String modelName = "";
protected OCRPredictorNative paddlePredictor = null;
protected float inferenceTime = 0;
// Only for object detection
protected Vector<String> wordLabels = new Vector<>();
protected int detLongSize = 960;
public float scoreThreshold = 0.1f;
protected Bitmap inputImage = null;
protected float preprocessTime = 0;
/**
* 自定义开关
*/
public boolean useSlim = true;
public boolean useOpencl = false;
public boolean checkModelLoaded = true;
public boolean runCls = false;
public boolean runDet = false;
public boolean runRec = true;
private static final String CHECK_IMG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAFQAAAA5CAYAAACoAQxFAAAAAXNSR0IArs4c6QAAAARzQklUCAgICHwIZIgAAAqMSURBVHic7ZtrUJTVH8c/uzyIxHpLLgUiKOM4kZYUqVM4I5lW5IsUm14x2kz2JotxhkVDMbSmbBvUmhhzAsRmEF5oeKlmGk0MzNKGmClDZfICm+AukagtsBf2+b9g9vQ87oV9lvX2n/28Ovucy3P2y9lzfpeDTpZlmQhhQ3+3J/D/RkTQMBMRNMxEBA0zEUHDTETQMBMRNMxEBA0zEUHDTETQMBMRNMxEBA0zEUHDjHS3J3CvcfPmTZxOJwDjx49HkrRJNOIKdbvdWCyW0GYXAo2NjRQUFFBQUMDZs2dHbH/x4kX++uuvsL3/448/ZuXKlaxcuZJLly6J54ODg0H1Dyi/2Wzm008/paurC5PJREpKiqr+xIkTDA0NaZpwdnY2cXFxfuvtdjt9fX0AuFwuv+2GhoZoaGigtraWxMRETCYTEyZM0DQXX+j1/60xt9uNy+Vi9+7dnD59mnfffZcpU6YE7B9Q0L1793Lu3DkAysrKMJlMTJo0SdRv374dh8OhacKff/55QEGD5dq1azQ0NOByuejq6uLDDz9ky5YtjBkzZlTjKgWVZZmmpiYOHToEgNFopLS0lMzMTP/9f/nlF7+Vb7/9NhkZGQBcvXqVLVu20N/fP6oJe3A6ndTW1lJbW8v58+c194+Pj8doNKLT6QD4448/qKioYLQJiKioKFGWZZlnn32WVatWAfDvv/+yYcMGTp486be/NG3aNL+VsbGxbNy4kaKiInp7e/nzzz8xmUxs3LhRtVnPnTuXF154we84LS0tfPPNN6pnTqeT+vp6ACZNmsTMmTMDflFfzJkzh4KCAr788ksAjh07Rnp6OsuWLdM8lgfPHwiGf/IA+fn5xMbGsnPnTlwuF1u3bmXHjh1Mnz7dq78UHx8f8AXx8fGUlpaybt067HY7LS0tfPbZZxQWFoo2SUlJPPXUU37H6O3t1fq9gmbFihW0t7fz888/A7B7927S0tJ44oknQhpPuUKV50NeXh6yLLNnzx6MRqNPMSFIOzQjI4OioiIAJkyYwLx581R/ybuJTqejsLCQ5ORkYPhnevTo0ZDHi46OFmXPCvXw0ksvUVVVFXDxBG1kzZ8/n/Xr15OZmak6mG7lxIkTDAwMIEkSubm5wQ4/KgwGA++88w4bNmygoKCA559/HoCTJ09y8OBBTWO1tbWJ8kgH0Pz58722F8loNPL6668HtYc988wzI7apqanBYrEQGxt7xwQFSE9Pp7q6mpiYGPGsr69PJVAoBOqfnp7u9Uw6d+5c0EbrvY5STABJkoiNjdU0xsDAgOpzoP6+TDQJ4OGHH/bZwWKxcObMGRYtWqRpUvcKS5YsYcmSJZr61NTUsH//fgCKi4tZsGCBpv5SdHQ0vk76o0ePsmvXLgYHB3E4HLz44ouaBr6dXLt2zeevKioqisTExFGNrTyU7Ha75v5SRkaGyjvwEBMTIya9c+dODAaD5r/W7aKyspKmpiav58nJyezatWtUYyu3jVAE1S9cuNBnxYIFC4SHIMsy5eXltLS0hDDF+wvlvqjVrQbQL1682G9lfn4+eXl5wLCR+8EHHwjf/m7yyCOPkJubS25uLk8//XRYxx47dqwoh+JmSyMFE9544w0sFgstLS04HA4qKysxmUw+t4k7xdKlS0XZarX69a2dTifff/99UGMaDAZycnJUp7rNZtM8txEN+6ioKIxGI+vWrePRRx/ltddeu6tiasFut1NRURFU29TU1DsjKEBcXBzl5eVedp4vPP6v0ie+n1CGFm/cuKG5f9CuZzBiAiJ9oDQ/7hZjx46lrKxM9Uz5+c033yQhIQH4z4A3GAyiPiRBbTZbWAK+Hjwn470gqCRJPPnkk6pnycnJdHV1AcPZg1tt8HHjxonyP//8o/md+pqamqAa/vTTT3R0dARs43a7he2q1eW7Vxg/fryIpPX29mpO8UjZ2dlBNfziiy/o6ekhLS2N7du3+1yBN27cEBHzyZMna5rIvYJerychIQGr1Yosy1y/fp0HH3ww+P7z5s0bsVF3dzc9PT3A8F7q7+fsSa7B/SsowEMPPSTKVqtVU9+g7B9lCCtQJLyzs1OUR8oO3m7cbndIZg+og0VaBQ3qlP/9999Fefbs2aK8Z88e4L8DSJlsO3XqFMuXLwdg0aJF5OTkAPDAAw9ommCoVFZWkpmZKd6rBWW63Gw2e9VbLBYSEhJ82uMjrlBZlmltbQWGT01lINpgMGAwGIiJicHlcvHDDz+Iura2Nqqrq5FlmejoaNH2TjgFdXV1HD58mIsXL4bUf+rUqaKsvOzgobq6mtWrV/PVV1951enXrFmjWoG30tXVJcyH2bNn+7VHm5ubuX79uupZQ0MDFRUVXrkZGLYC6urqqKur47nnnvP7fq18/fXX7N27FyBkQZWR+La2Nq/UtMViwWq1+ox46Ts6OryEUPLbb7+JclZWls82fX19VFdXi8+rVq0S9tx3331HeXm51y0QnU4nVu1oLyd4sFqtInwnSZII7ASL51CdPHmyiKvevHlTdTbIsizsWE9iUIkE+Awwe/j1119FWbl/enC5XGzbtk1MZu7cueTn55OVlUVJSQk2m42mpibsdjvFxcVhE8+DxzPzzAWGxSwpKfGZnVTalQcPHsThcNDZ2cnly5ex2Wzilkh2djbffvstMHyvIC0tDRgObnvSJL4yHQFTIA6HQwhqMBi8ctEul4tPPvlE7LFxcXGsXr0agOnTp/Pee+9RWlqKzWbj1KlTvP/++5SUlKhCZIEECiZVfetFsaioKCHm+fPn6e7u5sqVK1y5cgWz2ay6+HbgwAFV31svb3gEPXbsGMuWLUOn06lWq68knX7ixIl+L1mdPXtWuJJZWVmqA6W/v5+tW7dy/Phx8eWLi4tVNtyMGTPYvHmz8JpaW1vZvHlzQHPmwoULohyM+3rkyBFR1ul0GI1GsTIrKiooLy+nvr6e5uZmLl++HHAspe38+OOPM3HiRAA6Ojr48ccfAfUv1mfWM5BZ4Vl5MHztxYPZbOajjz5SuaJr1qzxaaPOnDmTsrIyNm3ahN1u58yZM7S2tpKTk8Phw4cZGBggOjqaoaEhLl26RHNzs+jrCVz4w263q+ZQWFioSnU/9thjPk/ppKQkpk2bxtSpU0lNTWXKlCmkpKSo3GVJkli+fLk4G7Zt20ZjYyOeu2Djxo3zuo0IIL388st+J3z69GlRnjVrFjAcMFi7dq3It+h0OtauXRswB5+ZmcmmTZsoKyvj1VdfFbZhe3u7WOG3MmvWrBFdvpiYGEpKSigqKuKVV17xys5mZWXR09MjhEtJSSElJSXglqMkLy+PI0eOYDabcTqdKj0WLlzo0wTU+ft/+cHBQdavX8+FCxdITEykqqpK1NXX11NbW0tcXBxGo9ErouMPs9lMamqq+Lxv3z7hHCiZM2cOb731VtAZzPb2dmbMmHFbrgd1d3dTWlqq2nvj4+PZsWOHz63Sr6DKAa9evaoymdxuN1VVVSxdutTvgRYMf//9N52dnej1eiRJYsyYMSQlJYXl4mw46e/vp7GxEbPZTGJiIosXL1aF+ZSMKGgEbdwfyaH7iIigYSYiaJiJCBpmIoKGmYigYeZ/jd/+RcTqcugAAAAASUVORK5CYII=";
private static final Bitmap checkingBitmap = BitmapFactory.decodeByteArray(Base64.decode(CHECK_IMG_BASE64, Base64.DEFAULT), 0, Base64.decode(CHECK_IMG_BASE64, Base64.DEFAULT).length);
/**
* 检测模型
*/
public String detModelFilename = "det_opt.nb";
/**
* 识别模型
*/
public String recModelFilename = "rec_opt.nb";
/**
* 文本方向检测模型
*/
public String clsModelFilename = "cls_opt.nb";
private final String defaultLabelPath = "labels/ppocr_keys_v1.txt";
private final String defaultModelPath = "models/ocr_v3_for_cpu";
/**
* slim模型 目前使用的是2.10版的opt工具转换的2.11版本不能正常使用
*/
private final String defaultModelPathSlim = "models/ocr_v3_for_cpu(slim)";
private int retryTime = 1;
public Predictor() {
}
public boolean init(Context appCtx) {
return this.init(appCtx, defaultModelPath, defaultLabelPath);
}
public boolean init(Context appCtx, boolean useSlim) {
if (this.isLoaded && this.useSlim == useSlim) {
return true;
}
this.useSlim = useSlim;
if (useSlim) {
return this.init(appCtx, defaultModelPathSlim, defaultLabelPath);
} else {
return this.init(appCtx, defaultModelPath, defaultLabelPath);
}
}
public boolean init(Context appCtx, String modelPath, String labelPath) {
isLoaded = loadModel(appCtx, modelPath, cpuThreadNum, cpuPowerMode);
if (!isLoaded) {
return false;
}
isLoaded = loadLabel(appCtx, labelPath);
if (!checkModelLoadedSuccess()) {
init(appCtx, modelPath, labelPath);
}
return isLoaded;
}
/**
* 初始化模型后通过识别预设图片校验是否初始化成功
* 曲线救国 深层的失败原因需要后续排查
*
* @return
*/
private boolean checkModelLoadedSuccess() {
if (!checkModelLoaded) {
return true;
}
if (!isLoaded) {
return false;
}
List<OcrResult> results = runOcr(checkingBitmap);
StringBuilder sb = new StringBuilder();
for (OcrResult result : results) {
sb.append(result.getLabel());
}
boolean check = sb.toString().contains("测试");
Log.d(TAG, "第" + retryTime + "次 校验是否初始化成功: " + check + " 识别结果:" + sb);
boolean result = check || retryTime++ >= 5;
if (!check && retryTime >= 5) {
Log.e(TAG, "初始化模型失败");
}
return result;
}
public boolean init(Context appCtx, String modelPath, String labelPath, int cpuThreadNum, String cpuPowerMode) {
isLoaded = loadModel(appCtx, modelPath, cpuThreadNum, cpuPowerMode);
if (!isLoaded) {
return false;
}
isLoaded = loadLabel(appCtx, labelPath);
return isLoaded;
}
public boolean init(Context appCtx, String modelPath, String labelPath, int cpuThreadNum, String cpuPowerMode,
int detLongSize, float scoreThreshold) {
boolean isLoaded = init(appCtx, modelPath, labelPath, cpuThreadNum, cpuPowerMode);
if (!isLoaded) {
return false;
}
this.detLongSize = detLongSize;
this.scoreThreshold = scoreThreshold;
return true;
}
protected boolean loadModel(Context appCtx, String modelPath, int cpuThreadNum, String cpuPowerMode) {
// Release model if exists
releaseModel();
// Load model
if (modelPath.isEmpty()) {
return false;
}
String realPath = modelPath;
if (modelPath.charAt(0) != '/') {
// Read model files from custom path if the first character of mode path is '/'
// otherwise copy model to cache from assets
realPath = appCtx.getCacheDir() + "/" + modelPath;
// region add by TonyJiangWJ
String key = "PADDLE_MODEL_LOADED";
// 进行了模型更新 需要强制覆盖旧模型
boolean loaded = PreferenceManager.getDefaultSharedPreferences(appCtx).getBoolean(key, false);
if (loaded) {
// 没有必要每次都复制
Utils.copyDirectoryFromAssetsIfNeeded(appCtx, modelPath, realPath);
} else {
Utils.copyDirectoryFromAssets(appCtx, modelPath, realPath);
PreferenceManager.getDefaultSharedPreferences(appCtx).edit().putBoolean(key, true).apply();
}
// endregion
}
OCRPredictorNative.Config config = new OCRPredictorNative.Config();
// 是否使用GPU
config.useOpencl = useOpencl ? 1 : 0;
config.cpuThreadNum = cpuThreadNum;
config.detModelFilename = realPath + File.separator + detModelFilename;
config.recModelFilename = realPath + File.separator + recModelFilename;
config.clsModelFilename = realPath + File.separator + clsModelFilename;
Log.i("Predictor", "model path" + config.detModelFilename + " ; " + config.recModelFilename + ";" + config.clsModelFilename);
config.cpuPower = cpuPowerMode;
paddlePredictor = new OCRPredictorNative(config);
this.cpuThreadNum = cpuThreadNum;
this.cpuPowerMode = cpuPowerMode;
this.modelPath = realPath;
this.modelName = realPath.substring(realPath.lastIndexOf("/") + 1);
return true;
}
public void releaseModel() {
if (paddlePredictor != null) {
paddlePredictor.destroy();
paddlePredictor = null;
}
isLoaded = false;
modelPath = "";
modelName = "";
}
protected boolean loadLabel(Context appCtx, String labelPath) {
wordLabels.clear();
wordLabels.add("black");
// Load word labels from file
try {
InputStream assetsInputStream = appCtx.getAssets().open(labelPath);
int available = assetsInputStream.available();
byte[] lines = new byte[available];
if (assetsInputStream.read(lines) <= 0) {
Log.e(TAG, "读取label失败");
return false;
}
assetsInputStream.close();
String words = new String(lines);
String[] contents = words.split("\n");
wordLabels.addAll(Arrays.asList(contents));
wordLabels.add(" ");
Log.i(TAG, "Word label size: " + wordLabels.size());
} catch (Exception e) {
Log.e(TAG, e.getMessage());
return false;
}
return true;
}
public List<OcrResult> runOcr(Bitmap inputImage) {
if (inputImage == null || !isLoaded()) {
return Collections.emptyList();
}
// 检测、分类、识别
int run_det = runDet ? 1 : 0, run_cls = runCls ? 1 : 0, run_rec = runRec ? 1 : 0;
// Warm up
for (int i = 0; i < warmupIterNum; i++) {
paddlePredictor.runImage(inputImage, detLongSize, run_det, run_cls, run_rec);
}
warmupIterNum = 0; // do not need warm
// Run inference
Date start = new Date();
ArrayList<OcrResultModel> results = paddlePredictor.runImage(inputImage, detLongSize, run_det, run_cls, run_rec);
Date end = new Date();
inferenceTime = (end.getTime() - start.getTime()) / (float) inferIterNum;
postProcess(results);
Log.i(TAG, "[stat] Preprocess Time: " + preprocessTime
+ " ; Inference Time: " + inferenceTime + " ;Box Size " + results.size());
List<OcrResult> ocrResults = new ArrayList<>();
for (OcrResultModel resultModel : results) {
Log.d(TAG, "recognize: " + resultModel.toString());
if (resultModel.getConfidence() >= scoreThreshold) {
ocrResults.add(new OcrResult(resultModel));
}
}
Collections.sort(ocrResults);
return ocrResults;
}
public boolean isLoaded() {
return paddlePredictor != null && isLoaded;
}
public String modelPath() {
return modelPath;
}
public String modelName() {
return modelName;
}
public int cpuThreadNum() {
return cpuThreadNum;
}
public String cpuPowerMode() {
return cpuPowerMode;
}
public float inferenceTime() {
return inferenceTime;
}
public Bitmap inputImage() {
return inputImage;
}
public float preprocessTime() {
return preprocessTime;
}
public String getDefaultLabelPath() {
return defaultLabelPath;
}
public String getDefaultModelPath() {
return defaultModelPath;
}
public String getDefaultModelPathSlim() {
return defaultModelPathSlim;
}
public boolean isUseSlim() {
return useSlim;
}
public void setInputImage(Bitmap image) {
if (image == null) {
return;
}
this.inputImage = image.copy(Bitmap.Config.ARGB_8888, true);
}
private void postProcess(ArrayList<OcrResultModel> results) {
for (OcrResultModel r : results) {
StringBuilder word = new StringBuilder();
for (int index : r.getWordIndex()) {
if (index >= 0 && index < wordLabels.size()) {
word.append(wordLabels.get(index).replace("\r", ""));
} else {
Log.e(TAG, "Word index is not in label list:" + index);
word.append(" ");
}
}
r.setLabel(word.toString());
r.setClsLabel(r.getClsIdx() == 1 ? "180" : "0");
}
}
}

View File

@@ -0,0 +1,199 @@
package com.baidu.paddle.lite.ocr;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.media.ExifInterface;
import android.os.Environment;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
/**
* @author PaddleOCR
* Modified by TonyJiangWJ
* @since 2023-08-06
*/
public class Utils {
private static final String TAG = Utils.class.getSimpleName();
public static void copyFileFromAssets(Context appCtx, String srcPath, String dstPath) {
if (srcPath.isEmpty() || dstPath.isEmpty()) {
return;
}
InputStream is = null;
OutputStream os = null;
try {
is = new BufferedInputStream(appCtx.getAssets().open(srcPath));
os = new BufferedOutputStream(new FileOutputStream(new File(dstPath)));
byte[] buffer = new byte[1024];
int length = 0;
while ((length = is.read(buffer)) != -1) {
os.write(buffer, 0, length);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
os.close();
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void copyDirectoryFromAssets(Context appCtx, String srcDir, String dstDir) {
if (srcDir.isEmpty() || dstDir.isEmpty()) {
return;
}
try {
if (!new File(dstDir).exists()) {
new File(dstDir).mkdirs();
}
for (String fileName : appCtx.getAssets().list(srcDir)) {
String srcSubPath = srcDir + File.separator + fileName;
String dstSubPath = dstDir + File.separator + fileName;
if (new File(srcSubPath).isDirectory()) {
copyDirectoryFromAssets(appCtx, srcSubPath, dstSubPath);
} else {
Log.d(TAG, "复制资源文件: " + srcSubPath + " => " + dstSubPath);
copyFileFromAssets(appCtx, srcSubPath, dstSubPath);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static void copyDirectoryFromAssetsIfNeeded(Context appCtx, String srcDir, String dstDir) {
if (srcDir.isEmpty() || dstDir.isEmpty()) {
return;
}
try {
if (!new File(dstDir).exists()) {
new File(dstDir).mkdirs();
}
for (String fileName : appCtx.getAssets().list(srcDir)) {
String srcSubPath = srcDir + File.separator + fileName;
String dstSubPath = dstDir + File.separator + fileName;
if (new File(srcSubPath).isDirectory()) {
copyDirectoryFromAssetsIfNeeded(appCtx, srcSubPath, dstSubPath);
} else {
if (new File(dstSubPath).exists()) {
return;
}
Log.d(TAG, "复制资源文件: " + srcSubPath + " => " + dstSubPath);
copyFileFromAssets(appCtx, srcSubPath, dstSubPath);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static float[] parseFloatsFromString(String string, String delimiter) {
String[] pieces = string.trim().toLowerCase().split(delimiter);
float[] floats = new float[pieces.length];
for (int i = 0; i < pieces.length; i++) {
floats[i] = Float.parseFloat(pieces[i].trim());
}
return floats;
}
public static long[] parseLongsFromString(String string, String delimiter) {
String[] pieces = string.trim().toLowerCase().split(delimiter);
long[] longs = new long[pieces.length];
for (int i = 0; i < pieces.length; i++) {
longs[i] = Long.parseLong(pieces[i].trim());
}
return longs;
}
public static String getSDCardDirectory() {
return Environment.getExternalStorageDirectory().getAbsolutePath();
}
public static boolean isSupportedNPU() {
return false;
// String hardware = android.os.Build.HARDWARE;
// return hardware.equalsIgnoreCase("kirin810") || hardware.equalsIgnoreCase("kirin990");
}
public static Bitmap resizeWithStep(Bitmap bitmap, int maxLength, int step) {
int width = bitmap.getWidth();
int height = bitmap.getHeight();
int maxWH = Math.max(width, height);
float ratio = 1;
int newWidth = width;
int newHeight = height;
if (maxWH > maxLength) {
ratio = maxLength * 1.0f / maxWH;
newWidth = (int) Math.floor(ratio * width);
newHeight = (int) Math.floor(ratio * height);
}
newWidth = newWidth - newWidth % step;
if (newWidth == 0) {
newWidth = step;
}
newHeight = newHeight - newHeight % step;
if (newHeight == 0) {
newHeight = step;
}
return Bitmap.createScaledBitmap(bitmap, newWidth, newHeight, true);
}
public static Bitmap rotateBitmap(Bitmap bitmap, int orientation) {
Matrix matrix = new Matrix();
switch (orientation) {
case ExifInterface.ORIENTATION_NORMAL:
return bitmap;
case ExifInterface.ORIENTATION_FLIP_HORIZONTAL:
matrix.setScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.setRotate(180);
break;
case ExifInterface.ORIENTATION_FLIP_VERTICAL:
matrix.setRotate(180);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_TRANSPOSE:
matrix.setRotate(90);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.setRotate(90);
break;
case ExifInterface.ORIENTATION_TRANSVERSE:
matrix.setRotate(-90);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.setRotate(-90);
break;
default:
return bitmap;
}
try {
Bitmap bmRotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
bitmap.recycle();
return bmRotated;
} catch (OutOfMemoryError e) {
e.printStackTrace();
return null;
}
}
}