update nlp

Author: crazyyanchao

nlp/dic/defined.dic

@@ -2,4 +2,5 @@
 知识图谱
 吖啶基氨酸基本基甲烷磺酰甲氧基苯胺
 进展
-有没有进展
+有没有进展
+复联终章快上映了好激动

nlp/nlp.properties

@@ -1,4 +1,4 @@
-# 自定义词典位置；动态增加的词对应词库也需要配置在该位置
+# 自定义词典位置；动态增加的词对应词库也需要配置在该位置[Hnlp/Thunlp/IK都可生效]
 defined.dic.path=nlp/dic/defined.dic
 # 停止词词库位置
 stopword.dic.path=nlp/dic/stopword.dic;nlp/dic/stopword.utf8

src/main/java/data/lab/ongdb/index/FreetextIK.java

@@ -50,5 +50,18 @@ public List<String> iKAnalyzer(@Name("text") String text, @Name("useSmart") bool
  return results;
  }
 
+ /**
+ * @param text:待分词文本
+ * @param useSmart:true 用智能分词，false 细粒度分词
+ * @return
+ * @Description: TODO(支持中英文本分词)
+ */
+ @UserFunction(name = "olab.nlp.ik")
+ @Description("IK分词器 - RETURN olab.nlp.ik({text},true) AS words")
+ public List<String> nlpIk(@Name("text") String text, @Name(value = "useSmart",defaultValue = "true") boolean useSmart) {
+ return iKAnalyzer(text, useSmart);
+ }
+
 }
 
+

src/main/java/data/lab/ongdb/inferencing/Inference.java

@@ -16,16 +16,14 @@
 import data.lab.ongdb.result.MapResult;
 import data.lab.ongdb.schema.auto.AutoCypher;
 import data.lab.ongdb.structure.AdjacencyNode;
+import data.lab.ongdb.util.ArrayUtils;
 import data.lab.ongdb.util.CypherBuilder;
 import data.lab.ongdb.util.FileUtil;
 import data.lab.wltea.analyzer.cfg.Configuration;
 import data.lab.wltea.analyzer.core.IKSegmenter;
 import data.lab.wltea.analyzer.core.Lexeme;
 import org.neo4j.graphdb.GraphDatabaseService;
-import org.neo4j.procedure.Context;
-import org.neo4j.procedure.Description;
-import org.neo4j.procedure.Name;
-import org.neo4j.procedure.UserFunction;
+import org.neo4j.procedure.*;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -37,7 +35,7 @@
 import java.util.stream.Stream;
 
 /**
- * 基于图数据的智能搜索
+ * 基于图数据上下文语义的智能搜索
  *
  * @author Yc-Ma
  * @PACKAGE_NAME: data.lab.ongdb.inferencing
@@ -72,13 +70,13 @@ public class Inference implements SemanticInter {
  @Description("RETURN olab.entity.recognition({graphDataSchema},{nodeHitsRules},{askString},{operator}) AS entityRecognitionHits")
  public Map<String, Object> entityRecognition(@Name("graphDataSchema") String graphDataSchema, @Name("nodeHitsRules") String nodeHitsRules, @Name("askString") String askString, @Name("operator") String operator, @Name(value = "words", defaultValue = "[]") List<String> words) throws IOException {
  if (words != null && !words.isEmpty()) {
- return executeEntityRecognition(graphDataSchema, nodeHitsRules, operator,words);
+ return executeEntityRecognition(graphDataSchema, nodeHitsRules, operator, words);
  } else {
  /*
  * 获取分词结果
  * */
  words = getSemanticSmartIKSegmenter(askString);
- return executeEntityRecognition(graphDataSchema, nodeHitsRules, operator,words);
+ return executeEntityRecognition(graphDataSchema, nodeHitsRules, operator, words);
  }
  }
 
@@ -763,6 +761,113 @@ private List<Map<String, Object>> executeIntentSchemaParse(String graphDataSchem
  }).collect(Collectors.toList());
  }
 
+ /**
+ * @param graphDataSchemaStr:图数据模型定义【schema主要定义标签和标签之间的关联类型】
+ * @param entityRecognitionHits :实体识别结果{semantic_schema}
+ * @param inferenceWeightStr :搜索本体的权重配置
+ * @return entityRecognitionHits:排序实体识别结果{semantic_schema}
+ * @Description: TODO(根据实体识别结果和本体权重配置 ， 对实体对进行排列组合 【 权重搜索队列 】 过程)
+ * //inferenceWeight格式【暂只支持LABEL配置】
+ * {
+ * "LABEL":
+ * {
+ * "label1": "weight",
+ * "label2": "weight"
+ * }
+ * }
+ */
+ @Override
+ @Procedure(name = "olab.entity.ptmd.queue")
+ @Description("RETURN olab.entity.ptmd.queue({graphDataSchemaStr},{entityRecognitionHits},{inferenceWeightStr}) AS entityRecognitionHits")
+ public Stream<MapResult> entityPtmdQueue(@Name("graphDataSchemaStr") String graphDataSchemaStr, @Name("entityRecognitionHits") Map<String, Object> entityRecognitionHits, @Name("inferenceWeightStr") String inferenceWeightStr) {
+
+ Map<String, Object> graphDataSchema = JSONObject.parseObject(graphDataSchemaStr);
+
+ List<Map<String, Object>> nodes = getGraphObject(graphDataSchema, "nodes");
+ //本体模型中所有节点标签，使用entities时node中labels需要和labels求交集
+ List<String> labels = nodes.stream().map(v -> {
+ List<String> list = (List<String>) v.get("labels");
+ return list.get(0);
+ }).collect(Collectors.toList());
+
+ Map<String, List<Map<String, Object>>> entities = (Map<String, List<Map<String, Object>>>) entityRecognitionHits.get("entities");
+
+ //将词填充到hits列表的每个对象
+ Set<String> words = entities.keySet();
+ for (String word : words) {
+ List<Map<String, Object>> list = entities.get(word);
+ List<Map<String, Object>> listReset = list.stream().peek(v -> v.put("word", word)).collect(Collectors.toList());
+ entities.put(word, listReset);
+ }
+
+ //一、不移除词，直接求笛卡尔积
+ List<List<Map<String, Object>>> descartesL = new ArrayList<>(new ArrayUtils().descartesStringKey(entities));
+
+ //二、循环移除一个词，求笛卡尔积
+ List<String> arrWords = Arrays.asList(words.toArray(new String[0]));
+ int size = arrWords.size();
+ for (int i = size - 1; i >= 0; i--) {
+ //移除一个词
+ Map<String, List<Map<String, Object>>> entitiesTemp = new HashMap<>(entities);
+ entitiesTemp.remove(arrWords.get(i));
+ //求笛卡尔积
+ List<List<Map<String, Object>>> descartesList = new ArrayUtils().descartesStringKey(entitiesTemp);
+ descartesL.addAll(descartesList);
+ }
+
+ //三、循环移除两个词，求笛卡尔积
+// List<String> arrWords = Arrays.asList(words.toArray(new String[0]));
+ for (int i = size - 1; i >= 0; i--) {
+ //移除两个词
+ Map<String, List<Map<String, Object>>> entitiesTemp = new HashMap<>(entities);
+ entitiesTemp.remove(arrWords.get(i));
+ if (i > 0) {
+ entitiesTemp.remove(arrWords.get(i - 1));
+ }
+ //求笛卡尔积
+ List<List<Map<String, Object>>> descartesList = new ArrayUtils().descartesStringKey(entitiesTemp);
+ descartesL.addAll(descartesList);
+ }
+
+ //笛卡尔积，组合列表，按照权重排序
+ List<List<Map<String, Object>>> descartesLSort = descartesLSort(descartesL, inferenceWeightStr, labels);
+
+ List<MapResult> entityRecognitionHitsList = new ArrayList<>();
+ for (List<Map<String, Object>> list : descartesLSort) {
+ Map<String, List<Map<String, Object>>> entitiesMap = new HashMap<>();
+ for (Map<String, Object> map : list) {
+ String word = String.valueOf(map.get("word"));
+ entitiesMap.put(word, new ArrayList<Map<String, Object>>() {{
+ add(map);
+ }});
+ }
+ entityRecognitionHitsList.add(new MapResult(new HashMap<String, Object>() {{
+ put("entities", entitiesMap);
+ }}));
+ }
+ return entityRecognitionHitsList.stream();
+ }
+
+ private List<List<Map<String, Object>>> descartesLSort(List<List<Map<String, Object>>> descartesL, String inferenceWeightStr, List<String> labels) {
+ JSONObject inferenceWeightObj = JSONObject.parseObject(inferenceWeightStr);
+ JSONObject labelsWeight = inferenceWeightObj.getJSONObject("LABEL");
+ return descartesL.stream().sorted((v1, v2) -> {
+ //计算weight
+ Integer v1f = v1.stream().map(v -> getWeight(labelsWeight, v, labels)).reduce((x, y) -> x += y).get();
+ Integer v2f = v2.stream().map(v -> getWeight(labelsWeight, v, labels)).reduce((x, y) -> x += y).get();
+ Integer v1Score = v1.size() + v1f;
+ Integer v2Score = v2.size() + v2f;
+ return v2Score.compareTo(v1Score);
+ }).collect(Collectors.toList());
+ }
+
+ private int getWeight(JSONObject labelsWeight, Map<String, Object> v, List<String> labels) {
+ List<String> labelsRe = new ArrayList<>(labels);
+ labelsRe.retainAll((List<String>) v.get("labels"));
+ Integer wei = labelsWeight.getInteger(labelsRe.size() > 0 ? labelsRe.get(0) : null);
+ return wei == null ? 0 : wei;
+ }
+
  /**
  * 执行QUERY
  **/

src/main/java/data/lab/ongdb/inferencing/SemanticInter.java

@@ -5,12 +5,15 @@
  *
  */
 
+import data.lab.ongdb.result.MapResult;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 /**
- * 基于图数据的智能搜索接口定义
+ * 基于图数据上下文语义的智能搜索接口定义
  *
  * @author Yc-Ma
  * @PACKAGE_NAME: data.lab.ongdb.inferencing
@@ -162,7 +165,7 @@ public interface SemanticInter {
  * @return 实体识别结果 关键词与hits列表 {entityRecognitionHits} 【包含node、label、relationship、relationshipType四个类型】
  * @Description: TODO
  */
- Map<String, Object> entityRecognition(String graphDataSchema, String nodeHitsRules, String askString, String operator,List<String> words) throws IOException;
+ Map<String, Object> entityRecognition(String graphDataSchema, String nodeHitsRules, String askString, String operator, List<String> words) throws IOException;
 
  /*
  * intentSchema入参格式样例：
@@ -269,30 +272,47 @@ public interface SemanticInter {
  * @param graphDataSchema:图数据Schema
  * @param query:查询语句【默认使用IK分词】
  * @param words:query分词结果【如果传入不为空的结果，则query参数失效】
- * @param intendedIntentStr:预期意图
- * [
- * {
- * "label": "LABEL1",
- * "sort":2, //意图标签返回优先级
- * "list": //意图模式识别可用关键词
- * [
- * ]
- * },
- * {
- * "label": "LABEL2",
- * "sort":1, //意图标签返回优先级
- * "list": //意图模式识别可用关键词
- * [
- * ]
- * }
- * ]
+ * @param intendedIntentStr:预期意图 [
+ * {
+ * "label": "LABEL1",
+ * "sort":2, //意图标签返回优先级
+ * "list": //意图模式识别可用关键词
+ * [
+ * ]
+ * },
+ * {
+ * "label": "LABEL2",
+ * "sort":1, //意图标签返回优先级
+ * "list": //意图模式识别可用关键词
+ * [
+ * ]
+ * }
+ * ]
  * @return
- * @Description: TODO(意图识别：模式匹配方法)
+ * @Description: TODO(意图识别 ： 模式匹配方法)
  */
- List<Map<String, Object>> intentSchemaParse(String graphDataSchema, String query,List<String> words, String intendedIntentStr) throws IOException;
+ List<Map<String, Object>> intentSchemaParse(String graphDataSchema, String query, List<String> words, String intendedIntentStr) throws IOException;
+
+ /**
+ * @param graphDataSchemaStr:图数据模型定义【schema主要定义标签和标签之间的关联类型】
+ * @param entityRecognitionHits:实体识别结果{semantic_schema}
+ * @param inferenceWeightStr:搜索本体的权重配置
+ * @return entityRecognitionHits:排序实体识别结果{semantic_schema}
+ * @Description: TODO(根据实体识别结果和本体权重配置，对实体对进行排列组合【权重搜索队列】过程)
+ * //inferenceWeight格式【暂只支持LABEL配置】
+ * {
+ * "LABEL":
+ * {
+ * "label1": "weight",
+ * "label2": "weight"
+ * }
+ * }
+ */
+ Stream<MapResult> entityPtmdQueue(String graphDataSchemaStr, Map<String, Object> entityRecognitionHits, String inferenceWeightStr);
 
 }
 
 
 
 
+

src/main/java/data/lab/ongdb/procedures/FunctionPartition.java

@@ -104,6 +104,17 @@ public List<List<Map<String, Object>>> cartesian(@Name("mapList") List<Map<Strin
  @Name("groupField") String groupField) {
  return new ArrayUtils().descartes(mapList, groupField);
  }
+
+ /**
+ * @param modelMap:【传入分好组的MAP】
+ * @return
+ * @Description: TODO(笛卡尔乘积算法 【 对列表中实体进行笛卡尔乘积运算进行组合 】)
+ */
+ @UserFunction(name = "olab.cartesian.byGroupMap")
+ @Description("笛卡尔乘积算法 【对列表中实体使用指定字段进行分组，并进行笛卡尔乘积运算进行组合】")
+ public List<List<Map<String, Object>>> cartesian(@Name("modelMap") Map<String, List<Map<String, Object>>> modelMap) {
+ return new ArrayUtils().descartesStringKey(modelMap);
+ }
 }
 
 

src/main/java/data/lab/ongdb/util/ArrayUtils.java

@@ -25,21 +25,19 @@ public class ArrayUtils {
  * @Description: TODO(笛卡尔乘积算法 【进行笛卡尔乘积运算进行组合】)
  */
  public List<List<Map<String, Object>>> descartes(Map<Object, List<Map<String, Object>>> modelMap) {
- /*
- * 按指定字段（type）分组
- * */
- Collection<List<Map<String, Object>>> mapValues = modelMap.values();
-
- /*
- * 原List
- * */
- List<List<Map<String, Object>>> dimensionValue = new ArrayList<>(mapValues);
+ List<List<Map<String, Object>>> result = new ArrayList<>();
+ new ArrayUtils().descartes(new ArrayList<>(modelMap.values()), result, 0, new ArrayList<>());
+ return result;
+ }
 
- /*
- * 返回集合
- * */
+ /**
+ * @param modelMap:【传入分好组的MAP】
+ * @return
+ * @Description: TODO(笛卡尔乘积算法 【进行笛卡尔乘积运算进行组合】)
+ */
+ public List<List<Map<String, Object>>> descartesStringKey(Map<String, List<Map<String, Object>>> modelMap) {
  List<List<Map<String, Object>>> result = new ArrayList<>();
- new ArrayUtils().descartes(dimensionValue, result, 0, new ArrayList<>());
+ new ArrayUtils().descartes(new ArrayList<>(modelMap.values()), result, 0, new ArrayList<>());
  return result;
  }
 

src/main/java/data/lab/wltea/analyzer/dic/Dictionary.java

@@ -23,6 +23,7 @@
  */
 package data.lab.wltea.analyzer.dic;
 
+import data.lab.ongdb.algo.nlp.NLPConfiguration;
 import data.lab.wltea.analyzer.cfg.Configuration;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
@@ -248,6 +249,8 @@ private List<String> getExtDictionarys() {
  }
  }
  }
+ List<String> nlpUserDic =NLPConfiguration.USER_DICT_LIST;
+ extDictFiles.addAll(nlpUserDic);
  return extDictFiles;
  }
 

wiki/3.5.x/OLAB-README-v-3.5.x.md

@@ -45,6 +45,7 @@ olab.moveDecimalPoint
 ### 中文分词 *-true 智能分词，false 细粒度分词
 ```cql
 RETURN olab.index.iKAnalyzer('复联终章快上映了好激动，据说知识图谱与人工智能技术应用到了那部电影！吖啶基氨基甲烷磺酰甲氧基苯胺是一种药嘛？',true) AS words
+RETURN olab.nlp.ik('复联终章快上映了好激动，据说知识图谱与人工智能技术应用到了那部电影！吖啶基氨基甲烷磺酰甲氧基苯胺是一种药嘛？') AS words
 ```
 - 组合切词结果后进行查询
 ```cql
@@ -707,13 +708,20 @@ UNWIND rels AS r
 CALL olab.schema.loop.vpath.ind(r,-1,vFMap) YIELD from,rel,to RETURN (from)-[rel]->(to) AS path,vFMap,graph,uniqueGraphID
 ```
 
+## 获取MAP的key列表
+```
+RETURN olab.map.keys({id:1,name:'a',type:1})
+```
+
 ## 使用笛卡尔乘积算法
 ```
 // 使用笛卡尔乘积算法 【对列表中实体使用指定字段进行分组，并进行笛卡尔乘积运算进行组合】
 // {mapList}:原List
 // {groupField}:列表中对象的分组字段
 RETURN olab.cartesian({mapList},{groupField}) AS cartesianList
 RETURN olab.cartesian([{id:1,name:'a',type:1},{id:2,name:'b',type:1},{id:3,name:'c',type:1},{id:4,name:'d',type:2},{id:5,name:'e',type:3},{id:6,name:'f',type:3}],'type') AS cartesianList
+RETURN olab.cartesian.byGroupMap({`1`:[1,2],`2`:[1,2,3],`3`:[4,5,6]})
+RETURN olab.cartesian.byGroupMap({a1:[1,2],a2:[1,2,3],a3:[4,5,6]})
 ```
 
 ## 提取图结构并以图搜图将结果转换为虚拟图