6.6.1 - Alpha2 - 支持 Node.js 拼音模块及结巴分词

jieba-analysis/src/main/java/com/huaban/analysis/jieba/CharacterUtil.kt

@@ -0,0 +1,35 @@
+package com.huaban.analysis.jieba
+
+import java.util.regex.Pattern
+
+object CharacterUtil {
+
+    var reSkip: Pattern = Pattern.compile("(\\d+\\.\\d+|[a-zA-Z0-9]+)")
+
+    private val connectors = charArrayOf('+', '#', '&', '.', '_', '-')
+
+    internal fun isChineseLetter(ch: Char) = ch.code in 0x4E00..0x9FA5
+
+    internal fun isEnglishLetter(ch: Char) = ch.code in 0x0041..0x005A || ch.code in 0x0061..0x007A
+
+    internal fun isDigit(ch: Char) = ch.code in 0x0030..0x0039
+
+    private fun isConnector(ch: Char) = connectors.contains(ch)
+
+    fun ccFind(ch: Char) = isChineseLetter(ch) || isEnglishLetter(ch) || isDigit(ch) || isConnector(ch)
+
+    /**
+     * 全角 to 半角,大写 to 小写
+     *
+     * @param input
+     * 输入字符
+     * @return 转换后的字符
+     */
+    fun regularize(input: Char) = when (input.code) {
+        12288 -> 32.toChar()
+        in 65281..65374 -> (input.code - 65248).toChar()
+        in 'A'.code..'Z'.code -> input + 32
+        else -> input
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/CharsDictionaryDatabase.kt

@@ -0,0 +1,19 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+
+open class CharsDictionaryDatabase private constructor(context: Context) : DictionaryDatabase(context) {
+
+    override val databaseName = "dict-chinese-chars.db"
+
+    companion object {
+
+        @Volatile
+        private var instance: CharsDictionaryDatabase? = null
+
+        fun getInstance(context: Context): CharsDictionaryDatabase = instance ?: synchronized(this) {
+            instance ?: CharsDictionaryDatabase(context.applicationContext).also { instance = it }
+        }
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/DictSegment.kt

@@ -0,0 +1,250 @@
+package com.huaban.analysis.jieba
+
+import java.util.*
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * 词典树分段，表示词典树的一个分枝
+ */
+internal class DictSegment(
+    // 当前节点上存储的字符
+    private val nodeChar: Char,
+) : Comparable<DictSegment> {
+
+    private val childrenSegment: MutableMap<Char, DictSegment> by lazy { ConcurrentHashMap() }
+
+    // 数组方式存储结构
+    private var childrenArray: Array<DictSegment?>? = null
+
+    // 当前节点存储的 Segment 数目
+    // storeSize <= ARRAY_LENGTH_LIMIT, 使用数组存储
+    // storeSize > ARRAY_LENGTH_LIMIT, 则使用 Map 存储
+    private var storeSize = 0
+
+    // 当前 DictSegment 状态
+    // 默认 0, 1 表示从根节点到当前节点的路径表示一个词
+    private var nodeState = 0
+
+    // 判断是否有下一个节点
+    private fun hasNextNode() = this.storeSize > 0
+
+    /**
+     * 匹配词段
+     */
+    @JvmOverloads
+    fun match(charArray: CharArray, begin: Int = 0, length: Int = charArray.size, searchHit: Hit? = null): Hit {
+        var niceSearchHit = searchHit
+        if (niceSearchHit == null) {
+            // 如果hit为空，新建
+            niceSearchHit = Hit()
+            // 设置hit的其实文本位置
+            niceSearchHit.begin = begin
+        } else {
+            // 否则要将HIT状态重置
+            niceSearchHit.setUnmatch()
+        }
+        // 设置hit的当前处理位置
+        niceSearchHit.end = begin
+
+        val keyChar = charArray[begin]
+        var ds: DictSegment? = null
+
+        // 引用实例变量为本地变量，避免查询时遇到更新的同步问题
+        val segmentArray = this.childrenArray
+        val segmentMap: Map<Char, DictSegment> = this.childrenSegment
+
+        // STEP1 在节点中查找keyChar对应的DictSegment
+        if (segmentArray != null) {
+            // 在数组中查找
+            val keySegment = DictSegment(keyChar)
+            val position = Arrays.binarySearch(segmentArray, 0, this.storeSize, keySegment)
+            if (position >= 0) {
+                ds = segmentArray[position]
+            }
+        } else {
+            // 在map中查找
+            ds = segmentMap[keyChar]
+        }
+
+        // STEP2 找到DictSegment，判断词的匹配状态，是否继续递归，还是返回结果
+        if (ds != null) {
+            if (length > 1) {
+                // 词未匹配完，继续往下搜索
+                return ds.match(charArray, begin + 1, length - 1, niceSearchHit)
+            } else if (length == 1) {
+                // 搜索最后一个char
+
+                if (ds.nodeState == 1) {
+                    // 添加HIT状态为完全匹配
+                    niceSearchHit.setMatch()
+                }
+                if (ds.hasNextNode()) {
+                    // 添加HIT状态为前缀匹配
+                    niceSearchHit.setPrefix()
+                    // 记录当前位置的DictSegment
+                    niceSearchHit.matchedDictSegment = ds
+                }
+                return niceSearchHit
+            }
+        }
+        // STEP3 没有找到DictSegment， 将HIT设置为不匹配
+        return niceSearchHit
+    }
+
+    /**
+     * 加载填充词典片段
+     */
+    private fun fillSegment(charArray: CharArray) {
+        this.fillSegment(charArray, 0, charArray.size, 1)
+    }
+
+    /**
+     * 屏蔽词典中的一个词
+     */
+    @Suppress("unused")
+    fun disableSegment(charArray: CharArray) {
+        this.fillSegment(charArray, 0, charArray.size, 0)
+    }
+
+    /**
+     * 加载填充词典片段
+     */
+    @Synchronized
+    private fun fillSegment(charArray: CharArray, begin: Int, length: Int, enabled: Int) {
+        // 获取字典表中的汉字对象
+        val beginChar = charArray[begin]
+        var keyChar = charMap[beginChar]
+        // 字典中没有该字，则将其添加入字典
+        if (keyChar == null) {
+            charMap[beginChar] = beginChar
+            keyChar = beginChar
+        }
+
+        // 搜索当前节点的存储，查询对应keyChar的keyChar，如果没有则创建
+        val ds = lookforSegment(keyChar, enabled)
+        if (ds != null) {
+            // 处理keyChar对应的segment
+            if (length > 1) {
+                // 词元还没有完全加入词典树
+                ds.fillSegment(charArray, begin + 1, length - 1, enabled)
+            } else if (length == 1) {
+                // 已经是词元的最后一个char,设置当前节点状态为enabled，
+                // enabled=1表明一个完整的词，enabled=0表示从词典中屏蔽当前词
+                ds.nodeState = enabled
+            }
+        }
+    }
+
+    /**
+     * 查找本节点下对应的keyChar的segment *
+     *
+     * @param keyChar
+     * @param create
+     * =1如果没有找到，则创建新的segment ; =0如果没有找到，不创建，返回null
+     * @return
+     */
+    private fun lookforSegment(keyChar: Char, create: Int): DictSegment? {
+        var ds: DictSegment? = null
+
+        if (this.storeSize <= ARRAY_LENGTH_LIMIT) {
+            // 获取数组容器，如果数组未创建则创建数组
+            val segmentArray = getChildrenArray()
+            // 搜寻数组
+            val keySegment = DictSegment(keyChar)
+            val position = Arrays.binarySearch(segmentArray, 0, this.storeSize, keySegment)
+            if (position >= 0) {
+                ds = segmentArray[position]
+            }
+
+            // 遍历数组后没有找到对应的segment
+            if (ds == null && create == 1) {
+                ds = keySegment
+                if (this.storeSize < ARRAY_LENGTH_LIMIT) {
+                    // 数组容量未满，使用数组存储
+                    segmentArray[storeSize] = ds
+                    // segment数目+1
+                    storeSize++
+                    Arrays.sort(segmentArray, 0, this.storeSize)
+                } else {
+                    // 数组容量已满，切换Map存储
+                    // 获取Map容器，如果Map未创建,则创建Map
+                    val segmentMap = getChildrenMap()
+                    // 将数组中的segment迁移到Map中
+                    migrate(segmentArray, segmentMap)
+                    // 存储新的segment
+                    segmentMap[keyChar] = ds
+                    // segment数目+1 ， 必须在释放数组前执行storeSize++ ， 确保极端情况下，不会取到空的数组
+                    storeSize++
+                    // 释放当前的数组引用
+                    this.childrenArray = null
+                }
+            }
+        } else {
+            // 获取Map容器，如果Map未创建,则创建Map
+            val segmentMap = getChildrenMap()
+            // 搜索Map
+            ds = segmentMap[keyChar]
+            if (ds == null && create == 1) {
+                // 构造新的segment
+                ds = DictSegment(keyChar)
+                segmentMap[keyChar] = ds
+                // 当前节点存储segment数目+1
+                storeSize++
+            }
+        }
+
+        return ds
+    }
+
+    /**
+     * 获取数组容器 线程同步方法
+     */
+    private fun getChildrenArray(): Array<DictSegment?> {
+        if (this.childrenArray == null) {
+            synchronized(this) {
+                if (this.childrenArray == null) {
+                    this.childrenArray = arrayOfNulls(ARRAY_LENGTH_LIMIT)
+                }
+            }
+        }
+        return childrenArray!!
+    }
+
+    private fun getChildrenMap(): MutableMap<Char, DictSegment> {
+        return childrenSegment
+    }
+
+    /**
+     * 将数组中的segment迁移到Map中以支持高性能存储
+     */
+    private fun migrate(segmentArray: Array<DictSegment?>, segmentMap: MutableMap<Char, DictSegment>) {
+        for (segment in segmentArray) {
+            if (segment != null) {
+                segmentMap[segment.nodeChar] = segment
+            }
+        }
+    }
+
+    /**
+     * 实现Comparable接口
+     */
+    override fun compareTo(other: DictSegment): Int {
+        // 对当前节点存储的char进行比较
+        return nodeChar.compareTo(other.nodeChar)
+    }
+
+    fun fillSegments(words: List<String>) {
+        for (word in words) {
+            fillSegment(word.toCharArray())
+        }
+    }
+
+    companion object {
+        // 公用字典表，存储汉字
+        private val charMap: MutableMap<Char, Char> = HashMap(16, 0.95f)
+
+        // 数组大小上限
+        private const val ARRAY_LENGTH_LIMIT = 3
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/DictionaryDatabase.kt

@@ -0,0 +1,80 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+import android.database.sqlite.SQLiteDatabase
+import java.io.Closeable
+import java.io.File
+import java.io.FileOutputStream
+import java.io.InputStream
+import java.security.MessageDigest
+import java.util.zip.GZIPInputStream
+
+abstract class DictionaryDatabase internal constructor(context: Context): Closeable {
+
+    abstract val databaseName: String
+
+    private var shouldForciblyCopyDatabase = false
+
+    private val compressedDatabaseName: String
+        get() = "$databaseName.gzip"
+
+    private val md5Key: String
+        get() = "database_md5_$databaseName"
+
+    val database: SQLiteDatabase by lazy {
+        copyDatabaseWithCompressionAndMd5(context)
+        SQLiteDatabase.openDatabase(context.getDatabasePath(databaseName).absolutePath, null, SQLiteDatabase.OPEN_READONLY)
+    }
+
+    override fun close() = database.close()
+
+    private fun copyDatabaseWithCompressionAndMd5(context: Context) {
+        val dbFile = File(context.getDatabasePath(databaseName).path)
+        val prefs = context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE)
+        val storedMd5 = prefs.getString(md5Key, null)
+        val currentMd5 by lazy { dbFile.md5() }
+
+        // 如果数据库文件存在，检查 MD5
+        if (dbFile.exists()) {
+            if (storedMd5 != null && storedMd5 == currentMd5) {
+                if (!shouldForciblyCopyDatabase) return // 文件有效，无需复制或解压
+            }
+        }
+
+        // 数据文件不存在或 MD5 不匹配，重新复制
+        dbFile.parentFile?.mkdirs()
+        context.assets.open(compressedDatabaseName).use { compressedInputStream ->
+            GZIPInputStream(compressedInputStream).use { gzipStream -> // 解压
+                FileOutputStream(dbFile).use { outputStream ->
+                    gzipStream.copyTo(outputStream)
+                }
+            }
+        }
+
+        // 保存 MD5 值
+        prefs.edit().putString(md5Key, currentMd5).apply()
+    }
+
+    companion object {
+        
+        private const val PREF_NAME = "dict_prefs"
+
+    }
+
+    /**
+     * 计算文件的 MD5
+     */
+    private fun File.md5(): String = inputStream().use { it.md5() }
+
+    private fun InputStream.md5(): String {
+        val md = MessageDigest.getInstance("MD5")
+        val buffer = ByteArray(1024)
+        var read: Int
+        while (this.read(buffer).also { read = it } != -1) {
+            md.update(buffer, 0, read)
+        }
+        val digest = md.digest()
+        return digest.joinToString("") { "%02x".format(it) }
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/Hit.java

@@ -0,0 +1,117 @@
+/**
+ * 
+ * IK 中文分词  版本 5.0
+ * IK Analyzer release 5.0
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * 源代码由林良益(linliangyi2005@gmail.com)提供
+ * 版权声明 2012，乌龙茶工作室
+ * provided by Linliangyi and copyright 2012 by Oolong studio
+ * 
+ */
+package com.huaban.analysis.jieba;
+
+/**
+ * 表示一次词典匹配的命中
+ */
+public class Hit {
+	//Hit不匹配
+	private static final int UNMATCH = 0x00000000;
+	//Hit完全匹配
+	private static final int MATCH = 0x00000001;
+	//Hit前缀匹配
+	private static final int PREFIX = 0x00000010;
+	
+	
+	//该HIT当前状态，默认未匹配
+	private int hitState = UNMATCH;
+	
+	//记录词典匹配过程中，当前匹配到的词典分支节点
+	private DictSegment matchedDictSegment; 
+	/*
+	 * 词段开始位置
+	 */
+	private int begin;
+	/*
+	 * 词段的结束位置
+	 */
+	private int end;
+	
+	
+	/**
+	 * 判断是否完全匹配
+	 */
+	public boolean isMatch() {
+		return (this.hitState & MATCH) > 0;
+	}
+	/**
+	 * 
+	 */
+	public void setMatch() {
+		this.hitState = this.hitState | MATCH;
+	}
+
+	/**
+	 * 判断是否是词的前缀
+	 */
+	public boolean isPrefix() {
+		return (this.hitState & PREFIX) > 0;
+	}
+	/**
+	 * 
+	 */
+	public void setPrefix() {
+		this.hitState = this.hitState | PREFIX;
+	}
+	/**
+	 * 判断是否是不匹配
+	 */
+	public boolean isUnmatch() {
+		return this.hitState == UNMATCH ;
+	}
+	/**
+	 * 
+	 */
+	public void setUnmatch() {
+		this.hitState = UNMATCH;
+	}
+	
+	public DictSegment getMatchedDictSegment() {
+		return matchedDictSegment;
+	}
+	
+	public void setMatchedDictSegment(DictSegment matchedDictSegment) {
+		this.matchedDictSegment = matchedDictSegment;
+	}
+	
+	public int getBegin() {
+		return begin;
+	}
+	
+	public void setBegin(int begin) {
+		this.begin = begin;
+	}
+	
+	public int getEnd() {
+		return end;
+	}
+	
+	public void setEnd(int end) {
+		this.end = end;
+	}	
+	
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/JiebaSegmenter.kt

@@ -0,0 +1,211 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+import com.huaban.analysis.jieba.viterbi.FinalSeg
+import java.util.*
+
+@Suppress("LocalVariableName")
+class JiebaSegmenter(context: Context) {
+
+    val dictionary by lazy { WordDictionary.getInstance(context) }
+
+    private val finalSeg = FinalSeg.instance
+
+    enum class SegMode { INDEX, SEARCH }
+
+    private fun createDAG(sentence: String): Map<Int, MutableList<Int>> {
+        val dag = HashMap<Int, MutableList<Int>>()
+        val trie = dictionary.trie
+        val chars: CharArray = sentence.toCharArray()
+        val N = chars.size
+        var i = 0
+        var j = 0
+        while (i < N) {
+            val hit = trie.match(chars, i, j - i + 1)
+            if (hit.isPrefix || hit.isMatch) {
+                if (hit.isMatch) {
+                    if (!dag.containsKey(i)) {
+                        val value: MutableList<Int> = ArrayList()
+                        dag[i] = value
+                        value.add(j)
+                    } else dag[i]!!.add(j)
+                }
+                j += 1
+                if (j >= N) {
+                    i += 1
+                    j = i
+                }
+            } else {
+                i += 1
+                j = i
+            }
+        }
+        i = 0
+        while (i < N) {
+            if (!dag.containsKey(i)) {
+                val value: MutableList<Int> = ArrayList()
+                value.add(i)
+                dag[i] = value
+            }
+            ++i
+        }
+        return dag
+    }
+
+    private fun calc(sentence: String, dag: Map<Int, MutableList<Int>>): Map<Int, Pair<Int>?> {
+        val N = sentence.length
+        val route = HashMap<Int, Pair<Int>?>()
+        route[N] = Pair(0, 0.0)
+        for (i in N - 1 downTo -1 + 1) {
+            var candidate: Pair<Int>? = null
+            for (x in dag[i]!!) {
+                val freq = dictionary.getFreq(sentence.substring(i, x + 1)) + route[x + 1]!!.freq
+                if (null == candidate) {
+                    candidate = Pair(x, freq)
+                } else if (candidate.freq < freq) {
+                    candidate.freq = freq
+                    candidate.key = x
+                }
+            }
+            route[i] = candidate
+        }
+        return route
+    }
+
+    fun process(paragraph: String, mode: SegMode): List<SegToken> {
+        val tokens: MutableList<SegToken> = ArrayList()
+        var sb = StringBuilder()
+        var offset = 0
+        for (i in paragraph.indices) {
+            val ch = CharacterUtil.regularize(paragraph[i])
+            when {
+                CharacterUtil.ccFind(ch) -> sb.append(ch)
+                else -> {
+                    if (sb.isNotEmpty()) {
+                        // process
+                        when (mode) {
+                            SegMode.SEARCH -> {
+                                for (word in sentenceProcess(sb.toString())) {
+                                    tokens.add(SegToken(word, offset, word.length.let { offset += it; offset }))
+                                }
+                            }
+                            else -> {
+                                for (token in sentenceProcess(sb.toString())) {
+                                    if (token.length > 2) {
+                                        var gram2: String?
+                                        var j = 0
+                                        while (j < token.length - 1) {
+                                            gram2 = token.substring(j, j + 2)
+                                            if (dictionary.containsWord(gram2)) tokens.add(SegToken(gram2, offset + j, offset + j + 2))
+                                            ++j
+                                        }
+                                    }
+                                    if (token.length > 3) {
+                                        var gram3: String?
+                                        var j = 0
+                                        while (j < token.length - 2) {
+                                            gram3 = token.substring(j, j + 3)
+                                            if (dictionary.containsWord(gram3)) tokens.add(SegToken(gram3, offset + j, offset + j + 3))
+                                            ++j
+                                        }
+                                    }
+                                    tokens.add(SegToken(token, offset, token.length.let { offset += it; offset }))
+                                }
+                            }
+                        }
+                        sb = StringBuilder()
+                        offset = i
+                    }
+                    if (dictionary.containsWord(paragraph.substring(i, i + 1))) tokens.add(SegToken(paragraph.substring(i, i + 1), offset, ++offset))
+                    else tokens.add(SegToken(paragraph.substring(i, i + 1), offset, ++offset))
+                }
+            }
+        }
+        if (sb.isNotEmpty()) when (mode) {
+            SegMode.SEARCH -> {
+                sentenceProcess(sb.toString()).mapTo(tokens) { token -> SegToken(token, offset, token.length.let { offset += it; offset }) }
+            }
+            else -> sentenceProcess(sb.toString()).forEach { token ->
+                if (token.length > 2) {
+                    var gram2: String?
+                    var j = 0
+                    while (j < token.length - 1) {
+                        gram2 = token.substring(j, j + 2)
+                        if (dictionary.containsWord(gram2)) tokens.add(SegToken(gram2, offset + j, offset + j + 2))
+                        ++j
+                    }
+                }
+                if (token.length > 3) {
+                    var gram3: String?
+                    var j = 0
+                    while (j < token.length - 2) {
+                        gram3 = token.substring(j, j + 3)
+                        if (dictionary.containsWord(gram3)) tokens.add(SegToken(gram3, offset + j, offset + j + 3))
+                        ++j
+                    }
+                }
+                tokens.add(SegToken(token, offset, token.length.let { offset += it; offset }))
+            }
+        }
+
+        return tokens
+    }
+
+    private fun sentenceProcess(sentence: String): List<String> {
+        val tokens: MutableList<String> = ArrayList()
+        val N = sentence.length
+        val dag = createDAG(sentence)
+        val route = calc(sentence, dag)
+
+        var x = 0
+        var y: Int
+        var buf: String
+        var sb = StringBuilder()
+        while (x < N) {
+            y = route[x]!!.key + 1
+            val lWord: String = sentence.substring(x, y)
+            when {
+                y - x == 1 -> sb.append(lWord)
+                else -> {
+                    if (sb.isNotEmpty()) {
+                        buf = sb.toString()
+                        sb = StringBuilder()
+                        when (buf.length) {
+                            1 -> tokens.add(buf)
+                            else -> when {
+                                dictionary.containsWord(buf) -> tokens.add(buf)
+                                else -> finalSeg.cut(buf, tokens)
+                            }
+                        }
+                    }
+                    tokens.add(lWord)
+                }
+            }
+            x = y
+        }
+        buf = sb.toString()
+        if (buf.isNotEmpty()) {
+            when (buf.length) {
+                1 -> tokens.add(buf)
+                else -> when {
+                    dictionary.containsWord(buf) -> tokens.add(buf)
+                    else -> finalSeg.cut(buf, tokens)
+                }
+            }
+        }
+        return tokens
+    }
+
+    fun cutSmall(hans: String, limit: Int): List<String> = when {
+        hans.isEmpty() || limit <= 0 -> emptyList()
+        else -> process(hans, SegMode.SEARCH)
+            .map { token -> token.word }
+            .flatMap { word ->
+                when {
+                    word.length > limit -> word.chunked(limit)
+                    else -> listOf(word)
+                }
+            }
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/Log.java

@@ -0,0 +1,24 @@
+package com.huaban.analysis.jieba;
+
+/**
+ * @description: enable output content to be controlled by switch
+ * @author: sharkdoodoo@foxmail.com
+ * @date: 2022/6/21
+ */
+public class Log {
+
+    private static final boolean LOG_ENABLE = Boolean.parseBoolean(System.getProperty("jieba.log.enable", "true"));
+
+    public static final void debug(String debugInfo) {
+        if (LOG_ENABLE) {
+            System.out.println(debugInfo);
+        }
+    }
+
+    public static final void error(String errorInfo) {
+        if (LOG_ENABLE) {
+            System.err.println(errorInfo);
+        }
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/Node.java

@@ -0,0 +1,13 @@
+package com.huaban.analysis.jieba;
+
+public class Node {
+
+    public Character value;
+    public Node parent;
+
+    public Node(Character value, Node parent) {
+        this.value = value;
+        this.parent = parent;
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/Pair.java

@@ -0,0 +1,20 @@
+package com.huaban.analysis.jieba;
+
+import org.jetbrains.annotations.NotNull;
+
+public class Pair<K> {
+    public K key;
+    public Double freq;
+
+    public Pair(K key, double freq) {
+        this.key = key;
+        this.freq = freq;
+    }
+
+    @NotNull
+    @Override
+    public String toString() {
+        return "Candidate [key=" + key + ", freq=" + freq + "]";
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/PhrasesDictionaryDatabase.kt

@@ -0,0 +1,20 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+
+open class PhrasesDictionaryDatabase private constructor(context: Context): DictionaryDatabase(context) {
+
+    override val databaseName = "dict-chinese-phrases.db"
+
+    companion object {
+
+        @Volatile
+        private var instance: PhrasesDictionaryDatabase? = null
+
+        fun getInstance(context: Context): PhrasesDictionaryDatabase = instance ?: synchronized(this) {
+            instance ?: PhrasesDictionaryDatabase(context.applicationContext).also { instance = it }
+        }
+
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/SegToken.java

@@ -0,0 +1,22 @@
+package com.huaban.analysis.jieba;
+
+public class SegToken {
+
+    public String word;
+
+    public int startOffset;
+
+    public int endOffset;
+
+    public SegToken(String word, int startOffset, int endOffset) {
+        this.word = word;
+        this.startOffset = startOffset;
+        this.endOffset = endOffset;
+    }
+
+    @Override
+    public String toString() {
+        return "[" + word + ", " + startOffset + ", " + endOffset + "]";
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/WordDictionary.kt

@@ -0,0 +1,94 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+import android.database.sqlite.SQLiteDatabase
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import kotlin.properties.Delegates
+
+class WordDictionary(context: Context) {
+
+    private val db: SQLiteDatabase by lazy { WordDictionaryDatabase.getInstance(context).database }
+    private var minFreq by Delegates.notNull<Double>()
+
+    internal var trie = DictSegment(0.toChar())
+        private set
+
+    init {
+        db.rawQuery("SELECT value FROM metadata WHERE `key` = 'min_freq' LIMIT 1", null).use {
+            minFreq = if (it.moveToFirst()) it.getDouble(0) else Double.MAX_VALUE
+        }
+        // db.rawQuery("SELECT word FROM dictionary", null).use { cursor ->
+        //     trie.fillSegments(generateSequence {
+        //         if (cursor.moveToNext()) cursor.getString(cursor.getColumnIndexOrThrow("word")) else null
+        //     })
+        // }
+        loadTrieConcurrently()
+    }
+
+    private fun loadTrieConcurrently() = runBlocking {
+        val chunkSize = 10000
+        val deferredResults = mutableListOf<Deferred<Unit>>()
+
+        db.rawQuery("SELECT word FROM dictionary", null).use { cursor ->
+            val words = mutableListOf<String>()
+            while (cursor.moveToNext()) {
+                val word = cursor.getString(cursor.getColumnIndexOrThrow("word"))
+                words.add(word)
+                if (words.size >= chunkSize) {
+                    val chunk = words.toList()
+                    words.clear()
+                    deferredResults.add(async(Dispatchers.Default) {
+                        trie.fillSegments(chunk)
+                    })
+                }
+            }
+            // 最后一块数据
+            if (words.isNotEmpty()) {
+                val chunk = words.toList()
+                deferredResults.add(async(Dispatchers.Default) {
+                    trie.fillSegments(chunk)
+                })
+            }
+        }
+
+        // 等待所有分块完成
+        deferredResults.awaitAll()
+    }
+
+    @Suppress("unused")
+    fun resetDict() {
+        trie = DictSegment(0.toChar())
+    }
+
+    fun containsWord(word: String?): Boolean {
+        if (word.isNullOrEmpty()) return false
+        db.rawQuery("SELECT 1 FROM dictionary WHERE word = ? LIMIT 1", arrayOf(word)).use {
+            return it.count > 0
+        }
+    }
+
+    fun getFreq(key: String?): Double {
+        if (!key.isNullOrEmpty()) {
+            db.rawQuery("SELECT normalized_freq FROM dictionary WHERE word = ? LIMIT 1", arrayOf(key)).use {
+                if (it.moveToFirst()) {
+                    return it.getDouble(it.getColumnIndexOrThrow("normalized_freq"))
+                }
+            }
+        }
+        return minFreq
+    }
+
+    companion object {
+        @Volatile
+        private var INSTANCE: WordDictionary? = null
+
+        fun getInstance(context: Context) = INSTANCE ?: synchronized(this) {
+            INSTANCE ?: WordDictionary(context.applicationContext).also { INSTANCE = it }
+        }
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/WordDictionaryDatabase.kt

@@ -0,0 +1,20 @@
+package com.huaban.analysis.jieba
+
+import android.content.Context
+
+open class WordDictionaryDatabase private constructor(context: Context): DictionaryDatabase(context) {
+
+    override val databaseName = "dict.db"
+
+    companion object {
+
+        @Volatile
+        private var instance: WordDictionaryDatabase? = null
+
+        fun getInstance(context: Context): WordDictionaryDatabase = instance ?: synchronized(this) {
+            instance ?: WordDictionaryDatabase(context.applicationContext).also { instance = it }
+        }
+
+    }
+
+}

jieba-analysis/src/main/java/com/huaban/analysis/jieba/viterbi/FinalSeg.kt

@@ -0,0 +1,201 @@
+package com.huaban.analysis.jieba.viterbi
+
+import com.huaban.analysis.jieba.CharacterUtil
+import com.huaban.analysis.jieba.Log
+import com.huaban.analysis.jieba.Node
+import com.huaban.analysis.jieba.Pair
+import java.io.BufferedReader
+import java.io.IOException
+import java.io.InputStreamReader
+import java.nio.charset.Charset
+import java.util.*
+
+class FinalSeg private constructor() {
+    private fun loadModel() {
+        val s = System.currentTimeMillis()
+        val prevStatus = HashMap<Char, CharArray>().also { prevStatus = it }
+        prevStatus['B'] = charArrayOf('E', 'S')
+        prevStatus['M'] = charArrayOf('M', 'B')
+        prevStatus['S'] = charArrayOf('S', 'E')
+        prevStatus['E'] = charArrayOf('B', 'M')
+
+        val start = HashMap<Char, Double>().also { start = it }
+        start['B'] = -0.26268660809250016
+        start['E'] = -3.14e+100
+        start['M'] = -3.14e+100
+        start['S'] = -1.4652633398537678
+
+        val trans = HashMap<Char, Map<Char, Double>>().also { trans = it }
+        val transB: MutableMap<Char, Double> = HashMap()
+        transB['E'] = -0.510825623765990
+        transB['M'] = -0.916290731874155
+        trans['B'] = transB
+        val transE: MutableMap<Char, Double> = HashMap()
+        transE['B'] = -0.5897149736854513
+        transE['S'] = -0.8085250474669937
+        trans['E'] = transE
+        val transM: MutableMap<Char, Double> = HashMap()
+        transM['E'] = -0.33344856811948514
+        transM['M'] = -1.2603623820268226
+        trans['M'] = transM
+        val transS: MutableMap<Char, Double> = HashMap()
+        transS['B'] = -0.7211965654669841
+        transS['S'] = -0.6658631448798212
+        trans['S'] = transS
+
+        val `is` = javaClass.getResourceAsStream(PROB_EMIT)!!
+        try {
+            val br = BufferedReader(InputStreamReader(`is`, Charset.forName("UTF-8")))
+            val emit = HashMap<Char, Map<Char, Double>>().also { emit = it }
+            var values: MutableMap<Char, Double>? = null
+            while (br.ready()) {
+                val line = br.readLine()
+                val tokens = line.split("\t".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
+                if (tokens.size == 1) {
+                    values = HashMap()
+                    emit[tokens[0][0]] = values
+                } else {
+                    values!![tokens[0][0]] = tokens[1].toDouble()
+                }
+            }
+        } catch (e: IOException) {
+            Log.error(String.format(Locale.getDefault(), "%s: load model failure!", PROB_EMIT))
+        } finally {
+            try {
+                `is`.close()
+            } catch (e: IOException) {
+                Log.error(String.format(Locale.getDefault(), "%s: close failure!", PROB_EMIT))
+            }
+        }
+        Log.debug(
+            String.format(
+                Locale.getDefault(), "model load finished, time elapsed %d ms.",
+                System.currentTimeMillis() - s
+            )
+        )
+    }
+
+    fun cut(sentence: String, tokens: MutableList<String>) {
+        var chinese = StringBuilder()
+        var other = StringBuilder()
+        for (element in sentence) {
+            if (CharacterUtil.isChineseLetter(element)) {
+                if (other.isNotEmpty()) {
+                    processOtherUnknownWords(other.toString(), tokens)
+                    other = StringBuilder()
+                }
+                chinese.append(element)
+            } else {
+                if (chinese.isNotEmpty()) {
+                    viterbi(chinese.toString(), tokens)
+                    chinese = StringBuilder()
+                }
+                other.append(element)
+            }
+        }
+        if (chinese.isNotEmpty()) viterbi(chinese.toString(), tokens)
+        else {
+            processOtherUnknownWords(other.toString(), tokens)
+        }
+    }
+
+    private fun viterbi(sentence: String, tokens: MutableList<String>) {
+        val v = Vector<MutableMap<Char, Double>>()
+        var path: MutableMap<Char?, Node?> = HashMap()
+
+        v.add(HashMap())
+        for (state in states) {
+            var emP = emit!![state]!![sentence[0]]
+            if (null == emP) emP = MIN_FLOAT
+            v[0][state] = start!![state]!! + emP
+            path[state] = Node(state, null)
+        }
+
+        for (i in 1..<sentence.length) {
+            val vv: MutableMap<Char, Double> = HashMap()
+            v.add(vv)
+            val newPath: MutableMap<Char?, Node?> = HashMap()
+            for (y in states) {
+                var emp = emit!![y]!![sentence[i]]
+                if (emp == null) emp = MIN_FLOAT
+                var candidate: Pair<Char>? = null
+                for (y0 in prevStatus!![y]!!) {
+                    var tranp = trans!![y0]!![y]
+                    if (null == tranp) tranp = MIN_FLOAT
+                    tranp += (emp + v[i - 1][y0]!!)
+                    if (null == candidate) candidate = Pair(y0, tranp)
+                    else if (candidate.freq <= tranp) {
+                        candidate.freq = tranp
+                        candidate.key = y0
+                    }
+                }
+                vv[y] = candidate!!.freq
+                newPath[y] = Node(y, path[candidate.key])
+            }
+            path = newPath
+        }
+        val probE = v[sentence.length - 1]['E']!!
+        val probS = v[sentence.length - 1]['S']!!
+        val posList = Vector<Char>(sentence.length)
+        var win: Node?
+        win = if (probE < probS) path['S']
+        else path['E']
+
+        while (win != null) {
+            posList.add(win.value)
+            win = win.parent
+        }
+        posList.reverse()
+
+        var begin = 0
+        var next = 0
+        for (i in sentence.indices) {
+            val pos = posList[i]
+            when (pos) {
+                'B' -> begin = i
+                'E' -> {
+                    tokens.add(sentence.substring(begin, i + 1))
+                    next = i + 1
+                }
+                'S' -> {
+                    tokens.add(sentence.substring(i, i + 1))
+                    next = i + 1
+                }
+            }
+        }
+        if (next < sentence.length) tokens.add(sentence.substring(next))
+    }
+
+    private fun processOtherUnknownWords(other: String, tokens: MutableList<String>) {
+        val mat = CharacterUtil.reSkip.matcher(other)
+        var offset = 0
+        while (mat.find()) {
+            if (mat.start() > offset) {
+                tokens.add(other.substring(offset, mat.start()))
+            }
+            tokens.add(mat.group())
+            offset = mat.end()
+        }
+        if (offset < other.length) tokens.add(other.substring(offset))
+    }
+
+    companion object {
+        private var singleInstance: FinalSeg? = null
+        private const val PROB_EMIT = "/prob_emit.txt"
+        private val states = charArrayOf('B', 'M', 'E', 'S')
+        private var emit: MutableMap<Char, Map<Char, Double>>? = null
+        private var start: MutableMap<Char, Double>? = null
+        private var trans: MutableMap<Char, Map<Char, Double>>? = null
+        private var prevStatus: MutableMap<Char, CharArray>? = null
+        private const val MIN_FLOAT = -3.14e100
+
+        @get:Synchronized
+        val instance: FinalSeg
+            get() {
+                if (singleInstance == null) {
+                    singleInstance = FinalSeg().apply { loadModel() }
+                }
+                return singleInstance!!
+            }
+    }
+}