Merge pull request #1429 from tarzanek/master

final fix #1376 symbols search works like full text search

Author: tarzanek

nbproject/project.properties

@@ -19,7 +19,7 @@ build.generated.dir=${build.dir}/generated
 run.classpath=\
  ${javac.classpath}:\
  ${build.classes.dir}
-lucene.version=6.4.0
+lucene.version=6.4.1
 lucene-core.jar=lucene-core-${lucene.version}.jar
 file.reference.opengrok-dev-test=test
 debug.test.classpath=${run.test.classpath}

opengrok-indexer/pom.xml

@@ -22,7 +22,7 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
 
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>org.opensolaris.opengrok</groupId>
  <artifactId>opengrok-indexer</artifactId>
@@ -45,27 +45,29 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
  <targetPath>org/opensolaris/opengrok/analysis/sql/</targetPath>
  <directory>../src/org/opensolaris/opengrok/analysis/sql/</directory>
  <includes>
- <include>*.dat</include>
+  <include>*.dat</include>
  </includes>
  </resource>
  <resource>
  <targetPath>org/opensolaris/opengrok/analysis/vb/</targetPath>
  <directory>../src/org/opensolaris/opengrok/analysis/vb/</directory>
  <includes>
- <include>*.dat</include>
+  <include>*.dat</include>
  </includes>
  </resource>
  <resource>
  <targetPath>org/opensolaris/opengrok/index/</targetPath>
  <directory>../src/org/opensolaris/opengrok/index/</directory>
- <excludes><exclude>*.java</exclude></excludes>
+ <excludes>
+ <exclude>*.java</exclude>
+ </excludes>
  </resource>
  <resource>
  <targetPath>org/opensolaris/opengrok/history/</targetPath>
  <directory>../src/org/opensolaris/opengrok/history/</directory>
  <excludes>
- <exclude>*.java</exclude>
- <exclude>*.html</exclude>
+  <exclude>*.java</exclude>
+  <exclude>*.html</exclude>
  </excludes>
  </resource>
  </resources>
@@ -74,17 +76,30 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
  <testResource>
  <targetPath>org/opensolaris/opengrok/history/</targetPath>
  <directory>../test/org/opensolaris/opengrok/history/</directory>
- <excludes><exclude>*.java</exclude></excludes>
+ <excludes>
+ <exclude>*.java</exclude>
+ </excludes>
  </testResource>
  <testResource>
  <targetPath>org/opensolaris/opengrok/analysis/php/</targetPath>
  <directory>../test/org/opensolaris/opengrok/analysis/php/</directory>
- <excludes><exclude>*.java</exclude></excludes>
+ <excludes>
+ <exclude>*.java</exclude>
+ </excludes>
  </testResource>
  <testResource>
  <targetPath>org/opensolaris/opengrok/analysis/haskell/</targetPath>
  <directory>../test/org/opensolaris/opengrok/analysis/haskell/</directory>
- <excludes><exclude>*.java</exclude></excludes>
+ <excludes>
+ <exclude>*.java</exclude>
+ </excludes>
+ </testResource>
+ <testResource>
+ <targetPath>org/opensolaris/opengrok/analysis/java/</targetPath>
+ <directory>../test/org/opensolaris/opengrok/analysis/java/</directory>
+ <excludes>
+ <exclude>*.java</exclude>
+ </excludes>
  </testResource>
  </testResources>
 
@@ -131,7 +146,7 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
  </execution>
  </executions>
  </plugin>
-<!-- TODO add the same fix as is in build.xml to patch jflex generated files to stop increasing buffer beyond token size that lucene accepts
+ <!-- TODO add the same fix as is in build.xml to patch jflex generated files to stop increasing buffer beyond token size that lucene accepts
  https://github.com/OpenGrok/OpenGrok/issues/1170 make parsers stop producing tokens > 32766 chars
  at least for PlainFullTokenizer, PlainSymbolTokenizer, JavaScriptSymbolTokenizer, JavaSymbolTokenizer
  use below ?
@@ -221,10 +236,12 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
  <groupId>org.apache.lucene</groupId>
  <artifactId>lucene-analyzers-common</artifactId>
  </dependency>
- <dependency> <groupId>org.apache.lucene</groupId>
+ <dependency> 
+ <groupId>org.apache.lucene</groupId>
  <artifactId>lucene-queryparser</artifactId>
  </dependency>
- <dependency> <groupId>org.apache.lucene</groupId>
+ <dependency> 
+ <groupId>org.apache.lucene</groupId>
  <artifactId>lucene-suggest</artifactId>
  </dependency>
  <dependency>
@@ -254,12 +271,12 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
  <artifactId>derby</artifactId>
  <scope>test</scope>
  </dependency>
-<!-- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derbyclient</artifactId>
- <scope>run</scope>
- </dependency>
--->
+ <!-- <dependency>
+  <groupId>org.apache.derby</groupId>
+  <artifactId>derbyclient</artifactId>
+  <scope>run</scope>
+  </dependency>
+ -->
 
  </dependencies>
 

opengrok-web-nbproject/nbproject/project.properties

@@ -35,7 +35,7 @@ excludes=
 file.reference.bcel-6.0.jar=../lib/bcel-6.0.jar
 file.reference.json-simple-1.1.1.jar=../lib/json-simple-1.1.1.jar
 j2ee.platform.classpath=${j2ee.server.home}/lib/annotations-api.jar:${j2ee.server.home}/lib/catalina-ant.jar:${j2ee.server.home}/lib/catalina-ha.jar:${j2ee.server.home}/lib/catalina-storeconfig.jar:${j2ee.server.home}/lib/catalina-tribes.jar:${j2ee.server.home}/lib/catalina.jar:${j2ee.server.home}/lib/ecj-4.4.2.jar:${j2ee.server.home}/lib/el-api.jar:${j2ee.server.home}/lib/jasper-el.jar:${j2ee.server.home}/lib/jasper.jar:${j2ee.server.home}/lib/jsp-api.jar:${j2ee.server.home}/lib/servlet-api.jar:${j2ee.server.home}/lib/tomcat-api.jar:${j2ee.server.home}/lib/tomcat-coyote.jar:${j2ee.server.home}/lib/tomcat-dbcp.jar:${j2ee.server.home}/lib/tomcat-i18n-es.jar:${j2ee.server.home}/lib/tomcat-i18n-fr.jar:${j2ee.server.home}/lib/tomcat-i18n-ja.jar:${j2ee.server.home}/lib/tomcat-jdbc.jar:${j2ee.server.home}/lib/tomcat-jni.jar:${j2ee.server.home}/lib/tomcat-util-scan.jar:${j2ee.server.home}/lib/tomcat-util.jar:${j2ee.server.home}/lib/tomcat-websocket.jar:${j2ee.server.home}/lib/websocket-api.jar
-lucene.version=6.4.0
+lucene.version=6.4.1
 lucene-core.jar=lucene-core-${lucene.version}.jar
 lucene-analyzers-common.jar=lucene-analyzers-common-${lucene.version}.jar
 lucene-queryparser.jar=lucene-queryparser-${lucene.version}.jar

platform/solaris/ips/create.sh

@@ -214,7 +214,7 @@ PKG pkgsend add file doc/ctags.config mode=0444 owner=root group=sys path=/usr/o
 
 
 # install libs
-LV=6.4.0
+LV=6.4.1
 for file in ant.jar \
  bcel-6.0.jar \
  lucene-analyzers-common-${LV}.jar \

platform/solaris/pkgdef/prototype

@@ -62,10 +62,10 @@ f none usr/opengrok/doc/opengrok-agent.properties=platform/solaris/default/openg
 f none usr/opengrok/lib/opengrok.jar=dist/opengrok.jar 0444 root bin
 f none usr/opengrok/lib/ant.jar=dist/lib/ant.jar 0444 root bin
 f none usr/opengrok/lib/bcel-6.0.jar=lib/bcel-6.0.jar 0444 root bin
-f none usr/opengrok/lib/lucene-core-6.4.0.jar=lib/lucene-core-6.4.0.jar 0444 root bin
-f none usr/opengrok/lib/lucene-analyzers-common-6.4.0.jar=lib/lucene-analyzers-common-6.4.0.jar 0444 root bin
-f none usr/opengrok/lib/lucene-queryparser-6.4.0.jar=lib/lucene-queryparser-6.4.0.jar 0444 root bin
-f none usr/opengrok/lib/lucene-suggest-6.4.0.jar=lib/lucene-suggest-6.4.0.jar 0444 root bin
+f none usr/opengrok/lib/lucene-core-6.4.1.jar=lib/lucene-core-6.4.1.jar 0444 root bin
+f none usr/opengrok/lib/lucene-analyzers-common-6.4.1.jar=lib/lucene-analyzers-common-6.4.1.jar 0444 root bin
+f none usr/opengrok/lib/lucene-queryparser-6.4.1.jar=lib/lucene-queryparser-6.4.1.jar 0444 root bin
+f none usr/opengrok/lib/lucene-suggest-6.4.1.jar=lib/lucene-suggest-6.4.1.jar 0444 root bin
 f none usr/opengrok/lib/jrcs.jar=lib/jrcs.jar 0444 root bin
 f none usr/opengrok/lib/swing-layout-0.9.jar=lib/swing-layout-0.9.jar 0444 root bin
 f none usr/opengrok/lib/json-simple-1.1.1.jar=lib/json-simple-1.1.1.jar 0444 root bin

pom.xml

@@ -58,7 +58,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
  </modules>
 
  <properties>
-<lucene.version>6.4.0</lucene.version>
+<lucene.version>6.4.1</lucene.version>
 <mavenjavadocplugin.version>2.10.4</mavenjavadocplugin.version>
  <compileSource>1.8</compileSource>
  <compileTarget>1.8</compileTarget>

src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java

@@ -37,18 +37,18 @@ public CompatibleAnalyser() {
  protected TokenStreamComponents createComponents(String fieldName) { 
  switch (fieldName) {
  case QueryBuilder.FULL:
- return new TokenStreamComponents(new PlainFullTokenizer());
+ return new TokenStreamComponents(new PlainFullTokenizer(FileAnalyzer.dummyReader));
  case QueryBuilder.REFS:
- return new TokenStreamComponents(new PlainSymbolTokenizer());
+ return new TokenStreamComponents(new PlainSymbolTokenizer(FileAnalyzer.dummyReader));
  case QueryBuilder.DEFS:
- return new TokenStreamComponents(new PlainSymbolTokenizer());
+ return new TokenStreamComponents(new PlainSymbolTokenizer(FileAnalyzer.dummyReader));
  case QueryBuilder.PATH:
  case QueryBuilder.PROJECT:
  return new TokenStreamComponents(new PathTokenizer());
  case QueryBuilder.HIST:
  return new HistoryAnalyzer().createComponents(fieldName);
  default:
- return new TokenStreamComponents(new PlainFullTokenizer());
+ return new TokenStreamComponents(new PlainFullTokenizer(FileAnalyzer.dummyReader));
  }
  }
 }

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -24,6 +24,8 @@
 package org.opensolaris.opengrok.analysis;
 
 import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
 import java.io.Writer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -140,13 +142,15 @@ public Genre getGenre() {
  return factory.getGenre();
  } 
 
+ public static Reader dummyReader=new StringReader("");
+ 
  /** Creates a new instance of FileAnalyzer
  * @param factory name of factory to be used 
  */
  public FileAnalyzer(FileAnalyzerFactory factory) {
  super(Analyzer.PER_FIELD_REUSE_STRATEGY);
  this.factory = factory; 
-  
+ SymbolTokenizer=new PlainSymbolTokenizer(dummyReader);
  }
 
  /**
@@ -180,23 +184,24 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc
  }
 
  // you analyzer HAS to override this to get proper symbols in results
- protected JFlexTokenizer SymbolTokenizer=new PlainSymbolTokenizer();
+ protected JFlexTokenizer SymbolTokenizer;
 
  @Override
  protected TokenStreamComponents createComponents(String fieldName) {
  switch (fieldName) {
  case "full":
- return new TokenStreamComponents(new PlainFullTokenizer());
+ return new TokenStreamComponents(new PlainFullTokenizer(dummyReader));
  case "path":
  case "project":
  return new TokenStreamComponents(new PathTokenizer());
  case "hist":
  return new HistoryAnalyzer().createComponents(fieldName);
+ //below is set by PlainAnalyzer to workaround #1376 symbols search works like full text search 
  case "refs": { 
- return new TokenStreamComponents(SymbolTokenizer); 
-  } 
+  return new TokenStreamComponents(SymbolTokenizer);
+ }
  case "defs":
- return new TokenStreamComponents(new PlainSymbolTokenizer());
+ return new TokenStreamComponents(new PlainSymbolTokenizer(dummyReader));
  default:
  LOGGER.log(
  Level.WARNING, "Have no analyzer for: {0}", fieldName);

src/org/opensolaris/opengrok/analysis/HistoryAnalyzer.java

@@ -58,7 +58,7 @@ public HistoryAnalyzer() {
  */
  @Override
  protected TokenStreamComponents createComponents(String fieldName) { 
- final PlainFullTokenizer plainfull = new PlainFullTokenizer();
+ final PlainFullTokenizer plainfull = new PlainFullTokenizer(FileAnalyzer.dummyReader);
  //we are counting position increments, this might affect the queries later and need to be in sync, especially for highlighting of results
  return new TokenStreamComponents(plainfull, new StopFilter(plainfull, stopWords));
  }

src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java

@@ -18,12 +18,13 @@
  */
 
 /*
- * Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2017, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.analysis;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.io.StringReader;
 import java.util.Stack;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -53,27 +54,25 @@ public abstract class JFlexTokenizer extends Tokenizer {
  abstract public void yybegin(int newState);
 
  abstract public int yystate(); 
- 
+  
  //TODO can be removed once we figure out jflex generation of empty constructor
  protected JFlexTokenizer(Reader in) {
  super();
+ setReader(in);
  }
- 
- protected JFlexTokenizer() {
- super();
- } 
-
+ 
  /**
  * Reinitialize the tokenizer with new reader.
  * @throws java.io.IOException in case of I/O error
  */
  @Override
  public void reset() throws IOException {
  super.reset();
- stack.clear();
+ stack.clear(); 
  this.yyreset(input);
+ clearAttributes();
  }
-
+ 
  @Override
  public final void close() throws IOException {
  super.close();
@@ -92,7 +91,8 @@ public final void close() throws IOException {
  * @throws IOException in case of I/O error
  */
  @Override
- public final boolean incrementToken() throws IOException {
+ public final boolean incrementToken() throws IOException { 
+ clearAttributes();
  return this.yylex();
  }