new scoring engines

Author: tarao

lib/lib.js

@@ -40,7 +40,7 @@ ExtractContentJS.Lib.Util = (function() {
  self.stop = function(name){ return self.get(name).stop(); };
  return self;
  };
- Util.Kind = function(word) {
+ Util.Token = function(word) {
  var regex = {
  hiragana: /[あ-んが-ぼぁ-ょゎっー]/,
  katakana: /[ア-ンガ-ボァ-ョヮッー]/,
@@ -63,7 +63,7 @@ ExtractContentJS.Lib.Util = (function() {
  first: tests(word.charAt(0)),
  last: tests(word.charAt(word.length-1))
  };
- self.tokenized = function(prev, next) {
+ self.isTokenized = function(prev, next) {
  var p = prev.length ? prev.charAt(prev.length-1) : '';
  var n = next.length ? next.charAt(0) : '';
  var check = function(w, test) {
@@ -76,6 +76,7 @@ ExtractContentJS.Lib.Util = (function() {
  };
  return check(p, self.first) && check(n, self.last);
  };
+
  return self;
  };
  Util.inherit = function(child,parent) {
@@ -101,15 +102,27 @@ ExtractContentJS.Lib.Util = (function() {
  Util.countMatchTokenized = function(text, word) {
  var count = 0;
  var prev = null;
- var kind = new Util.Kind(word);
+ var tok = new Util.Token(word);
  var texts = text.split(word);
  var len = texts.length;
  for (var i=0; i < len; i++) {
- if (prev && kind.tokenized(prev, texts[i])) count++;
+ if (prev && tok.isTokenized(prev, texts[i])) count++;
  prev = texts[i]
  }
  return count;
  };
+ Util.indexOfTokenized = function(text, word) {
+ var index = text.indexOf(word);
+ if (index >= 0) {
+ var tok = new Util.Token(word);
+ var p = index > 1 ? text.substr(index-1, 1) : '';
+ var n = text.substr(index+word.length, 1);
+ if (tok.isTokenized(p, n)) {
+ return index;
+ }
+ }
+ return -1;
+ };
  Util.dump = function(obj) {
  if (typeof obj == 'undefined') return 'undefined';
  if (typeof obj == 'string') return '"' + obj + '"';

lib/scoring-words.js

@@ -15,6 +15,7 @@ if (typeof ExtractContentJS == 'undefined') {
  if (typeof ns.RelativeWords.Engine != 'undefined') {
  return new ns.RelativeWords.Engine[name];
  }
+ return null;
  }
  };
 
@@ -52,12 +53,16 @@ if (typeof ExtractContentJS == 'undefined') {
  text: 32768
  }
  };
- var self = { weight: opt.weight || 0.3 };
+ var self = { weight: opt.weight || 0.4 };
 
  self.vote = function(doc, words) {
  var total = 0;
  var max = 0;
  var scores = {};
+ var content = (doc.content+'').substr(0, opt.limit.text);
+ content = content.toLowerCase();
+ var title = (doc.title||'').toLowerCase();
+ var url = (doc.url||'').toLowerCase();
  for (var t in words) total += words[t].df;
  for (var t in words) {
  var df = words[t].df;
@@ -67,20 +72,47 @@ if (typeof ExtractContentJS == 'undefined') {
 
  var tf = 0;
  var w = t.toLowerCase();
- var text = (doc.content+'').substr(0, opt.limit.text);
- tf += Util.countMatchTokenized(text.toLowerCase(), w);
- if (doc.title) {
- tf += Util.countMatchTokenized(doc.title.toLowerCase(), w);
- }
- if (doc.url) {
- tf += Util.countMatchTokenized(doc.url.toLowerCase(), w);
- }
+ tf += Util.countMatchTokenized(content, w);
+ tf += Util.countMatchTokenized(title, w);
+ tf += Util.countMatchTokenized(url, w);
 
  scores[t] = tf/idf;
  if (scores[t] > max) max = scores[t];
  }
  if (!max) return;
+ for (var t in scores) {
+ var score = scores[t] / max; // normalize
+ words[t].score += score * self.weight;
+ }
+ };
+
+ return self;
+ };
+
+ ns.RelativeWords.Engine.ContentPosition = function() {
+ var opt = arguments[0] || {
+ limit: {
+ text: 32768
+ }
+ };
+ var self = { weight: opt.weight || 0.1 };
+
+ self.vote = function(doc, words) {
+ var max = 0;
+ var scores = {};
+ var content = (doc.content+'').substr(0, opt.limit.text);
+ content = content.toLowerCase();
  for (var t in words) {
+ var w = t.toLowerCase();
+ var index = Util.indexOfTokenized(content, w);
+ if (index >= 0) {
+ scores[t] = scores[t] || 0;
+ scores[t] += 1.0 / (index+1);
+ if (max < scores[t]) max = scores[t];
+ }
+ }
+ if (!max) return;
+ for (var t in scores) {
  var score = scores[t] / max; // normalize
  words[t].score += score * self.weight;
  }
@@ -89,11 +121,46 @@ if (typeof ExtractContentJS == 'undefined') {
  return self;
  };
 
+ ns.RelativeWords.Engine.TitlePosition = function() {
+ var opt = arguments[0] || {
+ limit: {
+ text: 32768
+ }
+ };
+ var self = {
+ weight: {
+ global: (opt.weight && opt.weight.global) || 0.4,
+ title: (opt.weight && opt.weight.title) || 0.35
+ }
+ };
+
+ self.vote = function(doc, words) {
+ var max = 0;
+ var scores = {};
+ var title = (doc.title||'').toLowerCase();
+ for (var t in words) {
+ var w = t.toLowerCase();
+ var index = Util.indexOfTokenized(title, w);
+ if (index >= 0) {
+ scores[t] = 1 + self.weight.title / (1+Math.log(index+1));
+ if (max < scores[t]) max = scores[t];
+ }
+ }
+ if (!max) return;
+ for (var t in scores) {
+ var score = scores[t] / max; // normalize
+ words[t].score += score * self.weight.global;
+ }
+ };
+
+ return self;
+ };
+
  ns.suggestTags = function(url, title, body, tags) {
  var sc = new ns.RelativeWords();
  sc.addEngine( sc.factory.getEngine('TfIdf') );
-//  sc.addEngine( sc.factory.getEngine('Position') );
-//  sc.addEngine( sc.factory.getEngine('URL') );
+ sc.addEngine( sc.factory.getEngine('ContentPosition') );
+ sc.addEngine( sc.factory.getEngine('TitlePosition') );
  return sc.top({ url: url, title: title, content: body }, tags);
  };