train_limit and dev_limit, or split

Author: svlandeg

ml_datasets/loaders/dbpedia.py

@@ -11,14 +11,14 @@
 
 
 @register_loader("dbpedia")
-def dbpedia(loc=None, *, limit=0):
+def dbpedia(loc=None, *, train_limit=0, dev_limit=0):
  if loc is None:
  loc = get_file("dbpedia_csv", DBPEDIA_ONTOLOGY_URL, untar=True, unzip=True)
  train_loc = Path(loc) / "train.csv"
  test_loc = Path(loc) / "test.csv"
  return (
- read_dbpedia_ontology(train_loc, limit=limit),
- read_dbpedia_ontology(test_loc, limit=limit),
+ read_dbpedia_ontology(train_loc, limit=train_limit),
+ read_dbpedia_ontology(test_loc, limit=dev_limit),
  )
 
 

ml_datasets/loaders/imdb.py

@@ -8,12 +8,12 @@
 
 
 @register_loader("imdb")
-def imdb(loc=None, *, limit=0):
+def imdb(loc=None, *, train_limit=0, dev_limit=0):
  if loc is None:
  loc = get_file("aclImdb", IMDB_URL, untar=True, unzip=True)
  train_loc = Path(loc) / "train"
  test_loc = Path(loc) / "test"
- return read_imdb(train_loc, limit=limit), read_imdb(test_loc, limit=limit)
+ return read_imdb(train_loc, limit=train_limit), read_imdb(test_loc, limit=dev_limit)
 
 
 def read_imdb(data_dir, *, limit=0):

ml_datasets/spacy_readers.py

@@ -6,12 +6,12 @@
 
 
 def cmu_reader(
- path: Path = None, *, freq_cutoff: int = 0, limit: int = 0
+ path: Path = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
 ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
  from spacy.training.example import Example
 
- # Deduce the categories above threshold by inspecting all training data
- all_train_data, _ = list(cmu(path, limit=0))
+ # Deduce the categories above threshold by inspecting all data
+ all_train_data, _ = list(cmu(path, limit=0, split=1))
  counted_cats = {}
  for text, cats in all_train_data:
  for cat in cats:
@@ -20,7 +20,7 @@ def cmu_reader(
  unique_labels = [
  l for l in sorted(counted_cats.keys()) if counted_cats[l] >= freq_cutoff
  ]
- train_data, dev_data = cmu(path, limit=limit, shuffle=False, labels=unique_labels)
+ train_data, dev_data = cmu(path, limit=limit, shuffle=False, labels=unique_labels, split=split)
 
  def read_examples(data, nlp):
  for text, cats in data:
@@ -36,16 +36,16 @@ def read_examples(data, nlp):
 
 
 def dbpedia_reader(
- path: Path = None, *, limit: int = 0
+ path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
 ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
  from spacy.training.example import Example
 
- all_train_data, _ = dbpedia(path, limit=0)
+ all_train_data, _ = dbpedia(path, train_limit=0, dev_limit=1)
  unique_labels = set()
  for text, gold_label in all_train_data:
  assert isinstance(gold_label, str)
  unique_labels.add(gold_label)
- train_data, dev_data = dbpedia(path, limit=limit)
+ train_data, dev_data = dbpedia(path, train_limit=train_limit, dev_limit=dev_limit)
 
  def read_examples(data, nlp):
  for text, gold_label in data:
@@ -60,11 +60,11 @@ def read_examples(data, nlp):
 
 
 def imdb_reader(
- path: Path = None, *, limit: int = 0
+ path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
 ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
  from spacy.training.example import Example
 
- train_data, dev_data = imdb(path, limit=limit)
+ train_data, dev_data = imdb(path, train_limit=train_limit, dev_limit=dev_limit)
  unique_labels = ["pos", "neg"]
 
  def read_examples(data, nlp):