新增以文件方式构建敏感词树

wanghui03 · wanghui03 · commit b8d35fc1a917 · 2018-01-12T17:48:58.000+08:00
diff --git a/README.md b/README.md
@@ -24,7 +24,8 @@ php实现基于确定有穷自动机算法的铭感词过滤
     
     use DfaFilter\SensitiveHelper;
 
-### 获取敏感词库
+### 构建敏感词库树
+场景一: 可以拿到不同（用户）词库数组
 
     // 获取感词库索引数组
     $wordData = array(
@@ -35,21 +36,31 @@ php实现基于确定有穷自动机算法的铭感词过滤
         '成人卡通',
         ......
     );
+    
+    // get one helper
+    $handle = SensitiveHelper::init()->setTree($wordData);
+
+场景二: 全站使用一套敏感词库
+
+    // 获取感词库文件路径
+    $wordFilePath = 'tests/data/words.txt';
+    // get one helper
+    $handle = SensitiveHelper::init()->setTreeByFile($wordFilePath);
    
 ### 检测是否含有敏感词
 
-    $islegal = SensitiveHelper::init()->setTree($wordData)->islegal($content);
+    $islegal = $handle->islegal($content);
 ### 敏感词过滤
     
     // 敏感词替换为***为例
-    $filterContent = SensitiveHelper::init()->setTree($wordData)->replace($content, '***');
+    $filterContent = $handle->replace($content, '***');
     
 ### 获取文字中的敏感词
 
     // 获取内容中所有的敏感词
-    $sensitiveWordGroup = SensitiveHelper::init()->setTree($wordData)->getBadWord($content);
+    $sensitiveWordGroup = $handle->getBadWord($content);
     // 仅且获取一个敏感词
-    $sensitiveWordGroup = SensitiveHelper::init()->setTree($wordData)->getBadWord($content, 1);
+    $sensitiveWordGroup = $handle->getBadWord($content, 1);
 
 ### 如何使用单元测试进行测试
 #### 安装PHPUnit
diff --git a/src/DfaFilter/SensitiveHelper.php b/src/DfaFilter/SensitiveHelper.php
@@ -51,46 +51,45 @@ public static function init()
         return self::$_instance;
     }
 
+    /**
+     * 构建铭感词树【文件模式】
+     *
+     * @param string $sensitiveWord
+     * @return $this
+     */
+    public function setTreeByFile($filepath = '')
+    {
+        if (! file_exists($filepath)) {
+            throw new \Exception('词库文件不存在');
+        }
+
+        // 词库树初始化
+        $this->wordTree = new HashMap();
+
+        foreach ($this->yieldToReadFile($filepath) as $word) {
+            $this->buildWordToTree(trim($word));
+        }
+
+        return $this;
+    }
+
 
     /**
-     * 构建铭感词树
+     * 构建铭感词树【数组模式】
      *
      * @param string $sensitiveWord
      * @return $this
      */
-    public function setTree($sensitiveWords = '')
+    public function setTree($sensitiveWords = null)
     {
         if (empty($sensitiveWords)) {
             throw new \Exception('词库不能为空');
         }
-        $this->wordTree = new HashMap();
-        foreach ($sensitiveWords as $word) {
-            $tree = $this->wordTree;
-            $wordLength = mb_strlen($word, 'utf-8');
-            for ($i = 0; $i < $wordLength; $i++) {
-                $keyChar = mb_substr($word, $i, 1, 'utf-8');
-
-                // 获取子节点树结构
-                $tempTree = $tree->get($keyChar);
-
-                if ($tempTree) {
-                    $tree = $tempTree;
-                } else {
-                    // 设置标志位
-                    $newTree = new HashMap();
-                    $newTree->put('ending', false);
-
-                    // 添加到集合
-                    $tree->put($keyChar, $newTree);
-                    $tree = $newTree;
-                }
 
-                // 到达最后一个节点
-                if ($i == $wordLength - 1) {
-                    $tree->put('ending', true);
-                }
+        $this->wordTree = new HashMap();
 
-            }
+        foreach ($sensitiveWords as $word) {
+            $this->buildWordToTree($word);
         }
         return $this;
     }
@@ -117,21 +116,27 @@ public function getBadWord($content, $matchType = 1)
                 $nowMap = $tempMap->get($keyChar);
 
                 // 不存在节点树，直接返回
-                if (empty($nowMap)) break;
+                if (empty($nowMap)) {
+                    break;
+                }
 
                 // 存在，则判断是否为最后一个
                 $tempMap = $nowMap;
 
                 // 找到相应key，偏移量+1
-                $matchFlag ++;
+                $matchFlag++;
 
                 // 如果为最后一个匹配规则,结束循环，返回匹配标识数
-                if (false === $nowMap->get('ending')) continue;
+                if (false === $nowMap->get('ending')) {
+                    continue;
+                }
 
                 $flag = true;
 
                 // 最小规则，直接退出
-                if (1 === $matchType)  break;
+                if (1 === $matchType)  {
+                    break;
+                }
             }
 
             if (! $flag) {
@@ -204,16 +209,18 @@ public function islegal($content)
                 $nowMap = $tempMap->get($keyChar);
 
                 // 不存在节点树，直接返回
-                if (empty($nowMap)) break;
-
-                // 存在，则判断是否为最后一个
-                $tempMap = $nowMap;
+                if (empty($nowMap)) {
+                    break;
+                }
 
                 // 找到相应key，偏移量+1
-                $matchFlag ++;
+                $tempMap = $nowMap;
+                $matchFlag++;
 
                 // 如果为最后一个匹配规则,结束循环，返回匹配标识数
-                if (false === $nowMap->get('ending')) continue;
+                if (false === $nowMap->get('ending')) {
+                    continue;
+                }
 
                 return true;
             }
@@ -228,4 +235,49 @@ public function islegal($content)
         }
         return false;
     }
-}
+
+    protected function yieldToReadFile($filepath)
+    {
+        $fp = fopen($filepath, 'r');
+        while (! feof($fp)) {
+            yield fgets($fp);
+        }
+        fclose($fp);
+    }
+
+    // 将单个敏感词构建成树结构
+    protected function buildWordToTree($word = '')
+    {
+        if ('' === $word) {
+            return;
+        }
+        $tree = $this->wordTree;
+
+        $wordLength = mb_strlen($word, 'utf-8');
+        for ($i = 0; $i < $wordLength; $i++) {
+            $keyChar = mb_substr($word, $i, 1, 'utf-8');
+
+            // 获取子节点树结构
+            $tempTree = $tree->get($keyChar);
+
+            if ($tempTree) {
+                $tree = $tempTree;
+            } else {
+                // 设置标志位
+                $newTree = new HashMap();
+                $newTree->put('ending', false);
+
+                // 添加到集合
+                $tree->put($keyChar, $newTree);
+                $tree = $newTree;
+            }
+
+            // 到达最后一个节点
+            if ($i == $wordLength - 1) {
+                $tree->put('ending', true);
+            }
+        }
+
+        return;
+    }
+}
diff --git a/tests/ProTest.php b/tests/ProTest.php
@@ -0,0 +1,47 @@
+<?php
+/**
+ * Created by PhpStorm.
+ * User: zed
+ * Date: 17-11-13
+ * Time: 上午10:00
+ */
+
+use DfaFilter\SensitiveHelper;
+use PHPUnit\Framework\TestCase;
+
+class BaseTest extends TestCase
+{
+    protected $wordPoolPath;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        // 铭感词文件路径
+        $this->wordPoolPath = 'tests/data/words.txt';
+    }
+
+    public function testGetBadWord()
+    {
+        $content = '这是一段测试语句，请忽略赌球网';
+
+        // 过滤,其中【赌球网】在词库中
+        $filterContent = SensitiveHelper::init()
+            ->setTreeByFile($this->wordPoolPath)
+            ->getBadWord($content);
+
+        $this->assertEquals('赌球网', $filterContent[0]);
+    }
+
+    public function testFilterWord()
+    {
+        $content = '这是一段测试语句，请忽略赌球网';
+
+        // 过滤,其中【赌球网】在词库中
+        $filterContent = SensitiveHelper::init()
+            ->setTreeByFile($this->wordPoolPath)
+            ->replace($content,'*');
+
+        $this->assertEquals('这是一段测试语句，请忽略*', $filterContent);
+    }
+}
diff --git a/tests/data/words.txt b/tests/data/words.txt
