Pruning large domain variables

Author: natewoods-wf

net/__main__.py

@@ -43,8 +43,8 @@ def save(self):
  for key, value in self.data.iteritems()
  }
  print json.dumps(data, indent=4)
- with open(STORE, 'w') as file:
- json.dump(data, file, indent=4)
+ with open(STORE, 'w') as bitch:
+ json.dump(data, bitch, indent=4)
 
 
 class MyTCPHandler(BaseRequestHandler):

src/data/__init__.py

@@ -2,6 +2,9 @@
 from functools import partial
 
 
+DOMAIN_CAP = 5
+
+
 class DataSet(object):
  """
  A dataset (D) is a set of records (D_1 ... D_N), each of which is an
@@ -53,23 +56,32 @@ def __init__(self, name='scale'):
  item.append(variable.process(value))
  self.data.append(item)
 
- # Prune null variables
+ # Prune null variables (limit domains + delete if necessary)
  self.variables = [v for v in self.variables if v]
  [v.finish(self.data, idx) for idx, v in enumerate(self.variables)]
+ kill_idxs = list(reversed(sorted([
+ idx for idx, variable in enumerate(self.variables)
+ if len(variable.domain) > DOMAIN_CAP
+ ])))
+ print 'Killed variables', kill_idxs
+ for idx in kill_idxs:
+ del self.variables[idx]
  self.variable_map = {v.name: v for v in self.variables}
 
  # Order variables and data by domain size
  sorts = sorted(self.variables, key=lambda x: len(x.domain))[::-1]
  new_data = []
  for item in self.data:
+ for idx in kill_idxs:
+ del item[idx]
  new_item = [None] * len(item)
  for idx, variable in enumerate(self.variables):
  new_item[sorts.index(variable)] = item[idx]
  new_data.append(new_item)
  self.data = new_data
  self.variables = sorts
- # for variable in self.variables:
- #  print '{}: {}'.format(variable.name, repr(variable.domain))
+ for variable in self.variables:
+ print '{}: {}'.format(variable.name, repr(variable.domain))
 
 
 def is_num(test):
@@ -106,11 +118,9 @@ def process(self, value):
  return item
 
  def finish(self, data, idx):
- space = 5
- if len(self.domain) > space and all([is_num(x) for x in self.domain]):
+ if len(self.domain) > DOMAIN_CAP and all([is_num(x) for x in self.domain]):
  ints = [float(x) for x in self.domain]
- # self.var_type = binning(space, min(ints), max(ints))
- self.var_type = partial(bitchen_spaces_bro, space, min(ints), max(ints))
+ self.var_type = partial(bitchen_spaces_bro, DOMAIN_CAP, min(ints), max(ints))
  for item in data:
  item[idx] = self.var_type(item[idx])
  self.domain = set([self.var_type(x) for x in self.domain])

src/data/adult/__init__.py

@@ -64,17 +64,17 @@ def __call__(self, name):
  print 'Expanding Nodes'
  self.cap = log(2 * self.N / log(self.N))
  self.progress = Bar()
- self.progress.set_base(self.count_node_expansions(), True)
+ expansion_count = self.count_node_expansions()
+ self.progress.set_base(expansion_count, True)
  self.expand_ad_node(-1, set(), set(range(self.N)))
 
  print 'Prune Variables'
- self.progress.set_base(
- self.count_prune(set()) * len(self.variables), True
- )
+ prune_count = self.count_prune(set()) * len(self.variables)
+ self.progress.set_base(prune_count, True)
  for X in self.variables:
  self.prune(X, set(), self.score.get(X, set()))
 
- # Fuck if I know
+ # clear negative scores
  for key, value in self.score.cache.iteritems():
  if value < 0:
  self.score.cache[key] = 0
@@ -84,10 +84,12 @@ def __call__(self, name):
  with open(file_path, 'wb') as f:
  pickle.dump(self.score.cache, f)
  log_path = path.abspath(path.join(
- path.dirname(__file__), 'data', name, 'time.txt'
+ path.dirname(__file__), 'data', name, 'info.txt'
  ))
  with open(log_path, 'w') as f:
- f.write('{}s\n'.format(stop - start))
+ f.write('Time: {}s\nExpansions: {}\nPrunes: {}'.format(
+ stop - start, expansion_count, prune_count
+ ))
 
  return self.score
 
@@ -99,7 +101,7 @@ def count_node_expansions(self, i=-1, depth=0):
  self.variables.index(variable), depth + 1
  )
  size += count * len(variable.domain)
- return float(size)
+ return size
 
  def expand_ad_node(self, i, U, D_u):
  """
@@ -145,12 +147,12 @@ def update_scores(self, U, D_size):
  def count_prune(self, U):
  U = frozenset(U)
  if U in self._cache:
- return 1.0
+ return 1
  size = 1
  for X in self.vset.difference(U):
  size += self.count_prune(U.union({X}))
  self._cache.add(U)
- return float(size)
+ return size
 
  _cache2 = set()