Merge pull request FudanNLP#17 from tesseract2048/master

Optimize MurmurHash, benchmarked 33% faster Adding the serialVersionUID for incompatible problem.

Author: xpqiu

fnlp-core/src/main/java/org/fnlp/nlp/cn/tag/CWSTagger.java

@@ -229,8 +229,8 @@ public static void main(String[] args) throws Exception {
 HelpFormatter f = new HelpFormatter();
 f.printHelp(
 "SEG:\n"
-+ "java edu.fudan.nlp.tag.CWSTagger -f model_file input_file output_file;\n"
-+ "java edu.fudan.nlp.tag.CWSTagger -s model_file string_to_segement",
++ "java org.fnlp.tag.CWSTagger -f model_file input_file output_file;\n"
++ "java org.fnlp.tag.CWSTagger -s model_file string_to_segement",
 opt);
 return;
 }

fnlp-core/src/main/java/org/fnlp/util/MyStrings.java

@@ -1,21 +1,21 @@
 /**
-* This file is part of FNLP (formerly FudanNLP).
-* 
-* FNLP is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as published by
-* the Free Software Foundation, either version 3 of the License, or
-* (at your option) any later version.
-* 
-* FNLP is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-* GNU Lesser General Public License for more details.
-* 
-* You should have received a copy of the GNU General Public License
-* along with FudanNLP. If not, see <http://www.gnu.org/licenses/>.
-* 
-* Copyright 2009-2014 www.fnlp.org. All rights reserved. 
-*/
+ * This file is part of FNLP (formerly FudanNLP).
+ * 
+ * FNLP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * FNLP is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with FudanNLP. If not, see <http://www.gnu.org/licenses/>.
+ * 
+ * Copyright 2009-2014 www.fnlp.org. All rights reserved. 
+ */
 
 package org.fnlp.util;
 
@@ -116,9 +116,9 @@ public static String toString(float[] s, String delim) {
 sb.append(delim);
 }
 return sb.toString();
-
+
 }
-
+
 /**
  * 字符在字符串中出现的次数
  * 
@@ -155,8 +155,28 @@ public static String toString(float[][] s) {
 sb.append("\n");
 }
 return sb.toString();
+}
+/**
+ * 统计文本中某个字符串出现的次数
+ * @param str
+ * @param target
+ * @return
+ * 下午5:33:24
+ */
+public static int count(String str,String target){
+int count=0; 
+int index=0; 
+while(true){ 
+index=str.indexOf(target,index+1); 
+if(index>0){ 
+count++; 
+}else{ 
+break; 
+} 
+} 
+return count; 
 }
 
-
+
 
 }

fnlp-core/src/main/java/org/fnlp/util/Options.java

@@ -0,0 +1,32 @@
+/**
+ * 
+ */
+package org.fnlp.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * @author Xipeng Qiu E-mail: xpqiu@fudan.edu.cn
+ * @version 创建时间：2015年1月6日 下午4:39:23
+ * @since fnlp 2.1
+ */
+public class Options {
+HashMap<String,String> options = new HashMap<String,String>();
+ArrayList<String> rootArgs = new ArrayList<String>();
+
+
+public void parsing(String[] args) {
+
+for (int i=0; i<args.length; ++i) {
+if (args[i].charAt(0) != '-') {
+rootArgs.add(args[i]);
+} else if (i+1 < args.length) {
+options.put(args[i], args[++i]);
+} else {
+throw new IllegalArgumentException();
+}
+}
+}
+
+}

fnlp-core/src/main/java/org/fnlp/util/hash/MurmurHash.java

@@ -19,10 +19,7 @@
 
 package org.fnlp.util.hash;
 
-import java.io.UnsupportedEncodingException;
-
-
-/** 
+/**
  *
  * murmur hash 2.0.
  * 
@@ -40,12 +37,27 @@
  *
  */
 public final class MurmurHash extends AbstractHashCode {
+
+private static final long serialVersionUID = 4342869264396184799L;
 
 // all methods static; private constructor. 
 public MurmurHash() {}
 
+ protected byte[] toBytesWithoutEncoding(String str) {
+ int len = str.length();
+ int pos = 0;
+ byte[] buf = new byte[len << 1];
+ for (int i = 0; i < len; i++) {
+ char c = str.charAt(i);
+ buf[pos++] = (byte) (c & 0xFF);
+ buf[pos++] = (byte) (c >> 8);
+ }
+ return buf;
+ }
+
 public int hashcode(String str) {
-return hash32(str);
+ byte[] bytes = toBytesWithoutEncoding(str);
+ return hash32(bytes, bytes.length);
 }
 
 /** 
@@ -106,31 +118,6 @@ public int hash32( final byte[] data, int length) {
 }
 
 
-/** 
- * Generates 32 bit hash from a string.
- * 
- * @param text string to hash
- * @return 32 bit hash of the given string
- */
-public int hash32( final String text) {
-final byte[] bytes = text.getBytes(); 
-return hash32( bytes, bytes.length);
-}
-
-
-/** 
- * Generates 32 bit hash from a substring.
- * 
- * @param text string to hash
- * @param from starting index
- * @param length length of the substring to hash
- * @return 32 bit hash of the given string
- */
-public int hash32( final String text, int from, int length) {
-return hash32( text.substring( from, from+length));
-}
-
-
 /** 
  * Generates 64 bit hash from byte array of the given length and seed.
  * 
@@ -192,44 +179,4 @@ public long hash64( final byte[] data, int length) {
 return hash64( data, length, 0xe17a1465);
 }
 
-
-/** 
- * Generates 64 bit hash from a string.
- * 
- * @param text string to hash
- * @return 64 bit hash of the given string
- */
-public long hash64( final String text) {
-
-final byte[] bytes;
-try {
-bytes = text.getBytes("utf-8");
-
-return hash32( bytes, bytes.length);
-} catch (UnsupportedEncodingException e) {
-throw new RuntimeException("utf-8 encoding should be available on system.", e);
-}
-}
-
-
-
-/** 
- * Generates 64 bit hash from a substring.
- * 
- * @param text string to hash
- * @param from starting index
- * @param length length of the substring to hash
- * @return 64 bit hash of the given array
- */
-public long hash64( final String text, int from, int length) {
-
-final byte[] bytes;
-try {
-bytes = text.getBytes("utf-8");
-
-return hash64( bytes, bytes.length);
-} catch (UnsupportedEncodingException e) {
-throw new RuntimeException("utf-8 encoding should be available on system.", e);
-}
-}
-}
+}

fnlp-core/src/test/java/org/fnlp/util/hash/MurmurHashTest.java

@@ -0,0 +1,54 @@
+/**
+ * 
+ */
+package org.fnlp.util.hash;
+
+import org.fnlp.nlp.cn.tag.CWSTagger;
+import org.fnlp.util.exception.LoadModelException;
+
+/**
+ * @author Xipeng Qiu E-mail: xpqiu@fudan.edu.cn
+ * @version 创建时间：2015年1月8日 下午3:14:28
+ */
+public class MurmurHashTest {
+
+private CWSTagger cws;
+
+public MurmurHashTest() {
+try {
+long t = System.currentTimeMillis();
+cws = new CWSTagger("../models/seg.m");
+long elapsed = System.currentTimeMillis() - t;
+System.out.println("Fnlp loaded in " + elapsed + " ms.");
+} catch (LoadModelException e) {
+throw new RuntimeException("Failed to load fnlp", e);
+}
+}
+
+
+protected String[] getCws(String text) {
+return cws.tag2Array(text);
+}
+
+protected void benchmark() {
+long t = System.currentTimeMillis();
+String input = "12月1日，长江经济带海关区域通关一体化改革实现流域全覆盖，南昌、武汉、长沙、成都、重庆、贵阳、昆明等7个海关加入改革。当天，流域12个关区的海关特殊监管区域也纳入区域通关一体化，长江全流域真正实现了“12关如1关”。这标志着京津冀、长江经济带、广东地区三大区域通关一体化改革全面实施";
+for (int i = 0; i < 5000; i++) {
+getCws(input);
+}
+long elapsed = System.currentTimeMillis() - t;
+System.out.println("Benchmarked " + elapsed + " ms.");
+}
+
+public void run() {
+// warm up the code, and perform benchmark
+for (int k = 0; k < 10; k ++) {
+benchmark();
+}
+}
+
+public static void main(String[] args) throws InterruptedException {
+new MurmurHashTest().run();
+}
+}
+