improved HTML/XML parser, TREC 7 and 8

* wip: improved html parsing * character set detection from provided tags; cleanup; tests * tests for entity encodings * revert aquaint changes * revert formats/trec changes * customizable title tag * added trec disks 4&5 to show off new html parser * deprecating trec-robust04 * missed a spot * fixed recursive globs fixes #160 * working on documentation * various cleanup, metadata, and documentation

Author: seanmacavaney

ir_datasets/datasets/__init__.py

@@ -13,6 +13,7 @@
 from . import codec
 from . import cord19
 from . import cranfield
+from . import disks45
 from . import dpr_w100
 from . import codesearchnet
 from . import gov

ir_datasets/datasets/base.py

@@ -401,6 +401,14 @@ def documentation(self):
  return {}
 
 
+class Deprecated:
+ def __init__(self, message):
+ self._message = message
+
+ def deprecated(self):
+ return self._message
+
+
 class ExpectedFile:
  def __init__(self, path, expected_md5=None, instructions=None):
  self._path = Path(path)

ir_datasets/datasets/disks45.py

@@ -0,0 +1,102 @@
+import ir_datasets
+from ir_datasets.util import GzipExtract, TarExtract, Lazy, DownloadConfig
+from ir_datasets.formats import TrecQrels, TrecDocs, TrecQueries
+from ir_datasets.datasets.base import Dataset, FilteredQueries, FilteredQrels, YamlDocumentation
+
+
+NAME = 'disks45'
+
+
+QREL_DEFS = {
+ 2: 'highly relevant',
+ 1: 'relevant',
+ 0: 'not relevant',
+}
+
+QREL_DEFS_TREC78 = {
+ 1: 'relevant',
+ 0: 'not relevant',
+}
+
+DUA = ("Please confirm you agree to the TREC data usage agreement found at "
+ "<https://trec.nist.gov/data/cd45/index.html>")
+
+
+# folds from Huston & Croft 2014 <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.646.7749>
+ROBUST04_FOLDS = {
+ 'fold1': {'302', '303', '309', '316', '317', '319', '323', '331', '336', '341', '356', '357', '370', '373', '378', '381', '383', '392', '394', '406', '410', '411', '414', '426', '428', '433', '447', '448', '601', '607', '608', '612', '617', '619', '635', '641', '642', '646', '647', '654', '656', '662', '665', '669', '670', '679', '684', '690', '692', '700'},
+ 'fold2': {'301', '308', '312', '322', '327', '328', '338', '343', '348', '349', '352', '360', '364', '365', '369', '371', '374', '386', '390', '397', '403', '419', '422', '423', '424', '432', '434', '440', '446', '602', '604', '611', '623', '624', '627', '632', '638', '643', '651', '652', '663', '674', '675', '678', '680', '683', '688', '689', '695', '698'},
+ 'fold3': {'306', '307', '313', '321', '324', '326', '334', '347', '351', '354', '358', '361', '362', '363', '376', '380', '382', '396', '404', '413', '415', '417', '427', '436', '437', '439', '444', '445', '449', '450', '603', '605', '606', '614', '620', '622', '626', '628', '631', '637', '644', '648', '661', '664', '666', '671', '677', '685', '687', '693'},
+ 'fold4': {'320', '325', '330', '332', '335', '337', '342', '344', '350', '355', '368', '377', '379', '387', '393', '398', '402', '405', '407', '408', '412', '420', '421', '425', '430', '431', '435', '438', '616', '618', '625', '630', '633', '636', '639', '649', '650', '653', '655', '657', '659', '667', '668', '672', '673', '676', '682', '686', '691', '697'},
+ 'fold5': {'304', '305', '310', '311', '314', '315', '318', '329', '333', '339', '340', '345', '346', '353', '359', '366', '367', '372', '375', '384', '385', '388', '389', '391', '395', '399', '400', '401', '409', '416', '418', '429', '441', '442', '443', '609', '610', '613', '615', '621', '629', '634', '640', '645', '658', '660', '681', '694', '696', '699'}
+}
+
+
+def _init():
+ documentation = YamlDocumentation(f'docs/{NAME}.yaml')
+ base_path = ir_datasets.util.home_path()/NAME
+ dlc = DownloadConfig.context(NAME, base_path, dua=DUA)
+ subsets = {}
+
+ collection_nocr = TrecDocs(dlc['docs'],
+ path_globs=['**/FBIS/FB*', '**/FR94/??/FR*', '**/FT/*/FT*', '**/LATIMES/LA*'],
+ namespace=NAME,
+ lang='en',
+ expected_file_count=2295,
+ count_hint=ir_datasets.util.count_hint(NAME),
+ parser='sax',
+ docstore_path=base_path/'corpus.nocr.pklz4')
+
+ robust_queries = TrecQueries(GzipExtract(dlc['robust04-queries']), namespace=NAME, lang='en')
+ robust_qrels = TrecQrels(dlc['robust04-qrels'], QREL_DEFS)
+
+ base = Dataset(documentation('_'))
+
+ subsets['nocr'] = Dataset(
+ collection_nocr,
+ documentation('nocr'))
+
+ subsets['nocr/trec-robust-2004'] = Dataset(
+ collection_nocr,
+ robust_queries,
+ robust_qrels,
+ documentation('nocr/trec-robust-2004'))
+
+ for fold in ROBUST04_FOLDS:
+ qid_filter = make_filter(fold)
+ subsets[f'nocr/trec-robust-2004/{fold}'] = Dataset(
+ collection_nocr,
+ FilteredQueries(robust_queries, qid_filter),
+ FilteredQrels(robust_qrels, qid_filter),
+ documentation(f'nocr/trec-robust-2004/{fold}'))
+
+ subsets['nocr/trec8'] = Dataset(
+ collection_nocr,
+ TrecQrels(TarExtract(dlc['trec8-qrels'], 'qrels.trec8.adhoc.parts1-5'), QREL_DEFS_TREC78),
+ TrecQueries(GzipExtract(dlc['trec8-queries']), namespace=NAME, lang='en'),
+ documentation('nocr/trec8'))
+
+ subsets['nocr/trec7'] = Dataset(
+ collection_nocr,
+ TrecQrels([
+ GzipExtract(TarExtract(dlc['trec7-qrels'], 'qrels.trec7.adhoc.part1.gz')),
+ GzipExtract(TarExtract(dlc['trec7-qrels'], 'qrels.trec7.adhoc.part2.gz')),
+ GzipExtract(TarExtract(dlc['trec7-qrels'], 'qrels.trec7.adhoc.part3.gz')),
+ GzipExtract(TarExtract(dlc['trec7-qrels'], 'qrels.trec7.adhoc.part4.gz')),
+ GzipExtract(TarExtract(dlc['trec7-qrels'], 'qrels.trec7.adhoc.part5.gz')),
+ ], QREL_DEFS_TREC78),
+ TrecQueries(GzipExtract(dlc['trec7-queries']), namespace=NAME, lang='en'),
+ documentation('nocr/trec7'))
+
+ ir_datasets.registry.register(NAME, base)
+ for s in sorted(subsets):
+ ir_datasets.registry.register(f'{NAME}/{s}', subsets[s])
+
+ return base, subsets
+
+
+def make_filter(fold):
+ return Lazy(lambda: ROBUST04_FOLDS[fold])
+
+
+base, subsets = _init()

ir_datasets/datasets/trec_robust04.py

@@ -1,7 +1,7 @@
 import ir_datasets
 from ir_datasets.util import GzipExtract, Lazy, DownloadConfig
 from ir_datasets.formats import TrecQrels, TrecDocs, TrecQueries
-from ir_datasets.datasets.base import Dataset, FilteredQueries, FilteredQrels, YamlDocumentation
+from ir_datasets.datasets.base import Dataset, FilteredQueries, FilteredQrels, YamlDocumentation, Deprecated
 
 
 NAME = 'trec-robust04'
@@ -27,6 +27,9 @@
 }
 
 
+DEPRECATED_MESSAGE = '{} is deprecated. Consider using {} instead, which provides better parsing of the corpus.'
+
+
 def _init():
  documentation = YamlDocumentation(f'docs/{NAME}.yaml')
  base_path = ir_datasets.util.home_path()/NAME
@@ -38,15 +41,21 @@ def _init():
  queries = TrecQueries(GzipExtract(dlc['queries']), namespace=NAME, lang='en')
  qrels = TrecQrels(dlc['qrels'], QREL_DEFS)
 
- base = Dataset(collection, queries, qrels, documentation('_'))
+ base = Dataset(
+ collection,
+ queries,
+ qrels,
+ documentation('_'),
+ Deprecated(DEPRECATED_MESSAGE.format(NAME, f'disks45/nocr/trec-robust-2004')))
 
  for fold in FOLDS:
  qid_filter = make_filter(fold)
  subsets[fold] = Dataset(
  FilteredQueries(queries, qid_filter),
  FilteredQrels(qrels, qid_filter),
  collection,
- documentation(fold))
+ documentation(fold),
+ Deprecated(DEPRECATED_MESSAGE.format(f'{NAME}/{fold}', f'disks45/nocr/trec-robust-2004/{fold}')))
 
  ir_datasets.registry.register(NAME, base)
  for s in sorted(subsets):

ir_datasets/docs/bibliography.bib

@@ -867,6 +867,28 @@ @inproceedings{petroni-etal-2021-kilt
  pages = "2523--2544",
 }
 
+@misc{Voorhees1996Disks45,
+ title = {NIST TREC Disks 4 and 5: Retrieval Test Collections Document Set},
+ author = {Ellen M. Voorhees},
+ doi = {10.18434/t47g6m},
+ year = {1996},
+ publisher = {National Institute of Standards and Technology}
+}
+
+@inproceedings{Voorhees1998Trec7,
+ title = {Overview of the Seventh Text Retrieval Conference (TREC-7)},
+ author = {Ellen M. Voorhees and Donna Harman},
+ year = {1998},
+ booktitle = {TREC}
+}
+
+@inproceedings{Voorhees1999Trec8,
+ title = {Overview of the Eight Text Retrieval Conference (TREC-8)},
+ author = {Ellen M. Voorhees and Donna Harman},
+ year = {1999},
+ booktitle = {TREC}
+}
+
 @article{Lawrie2022HC4,
  author = {Dawn Lawrie and James Mayfield and Douglas W. Oard and Eugene Yang},
  title = {HC4: A New Suite of Test Collections for Ad Hoc CLIR},

ir_datasets/docs/disks45.yaml

@@ -0,0 +1,126 @@
+_:
+ pretty_name: 'TREC Disks 4 and 5'
+ desc: '
+<p>
+TREC Disks 4 and 5, including documents from the Financial Times, the Congressional Record,
+the Federal Register, the Foreign Broadcast Information Service, and the Los Angeles Times.
+</p>
+<p>
+This dataset is a placeholder for the complete collection, but at this time, only the version
+of the dataset without the Congressional Record (<a class="ds-ref">disks45/nocr</a>) are provided.
+</p>
+<ul>
+<li><a href="https://trec.nist.gov/data/cd45/">Information and access to TREC Disks 4 and 5.</a></li>
+</ul>
+'
+ docs_instructions: &inst "docs available from NIST"
+ data_access: '
+<p>
+To use this dataset, you need a copy of <a href="https://trec.nist.gov/data/cd45/index.html">TREC
+Disks 4 and 5</a>, provided by NIST.
+</p>
+<p>
+Your organization may already have a copy. If this is the case, you may only need to complete a new
+"Individual Argeement". Otherwise, your organization will need to file the "Organizational agreement"
+with NIST. It can take some time to process, but you will end up with a password-protected download link.
+</p>
+<p>
+ir_datasets needs the following directories from the source:
+</p>
+<ul>
+<li><kbd>FBIS</kbd></li>
+<li><kbd>FR94</kbd></li>
+<li><kbd>FT</kbd></li>
+<li><kbd>LATIMES</kbd></li>
+</ul>
+<p>
+ir_datasets expects the above directories to be copied/linked under <kbd>~/.ir_datasets/disks45/corpus</kbd>.
+The source document files themselves can either be compressed or uncompressed (it seems they have been distributed
+both ways in the past.) If ir_datasets does not find the files it is expecting, it will raise an error.
+</p>
+'
+
+nocr:
+ desc: '
+<p>
+A version of <a class="ds-ref">disks45</a> without the Congressional Record. This is the typical setting for
+tasks like TREC 7, TREC 8, and TREC Robust 2004.
+</p>
+'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45']
+
+nocr/trec-robust-2004:
+ desc: '
+<p>
+The TREC Robust retrieval task focuses on "improving the consistency of retrieval technology by
+focusing on poorly performing topics."
+</p>
+<p>
+The TREC Robust document collection is from TREC disks 4 and 5. Due to the
+copyrighted nature of the documents, this collection is for research use only, which requires
+agreements to be filed with NIST. See details <a href="https://trec.nist.gov/data/cd45/index.html">here</a>.
+</p>
+<ul>
+<li>Documents: News articles</li>
+<li>Queries: keyword queries, descriptions, narratives</li>
+<li>Relevance: Deep judgments</li>
+<li><a href="https://trec.nist.gov/pubs/trec13/papers/ROBUST.OVERVIEW.pdf">Task Overview Paper</a></li>
+<li>See also: <a class="ds-ref">aquaint/trec-robust-2005</a></li>
+</ul>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec-robust-2004/fold1:
+ desc: '
+<p>Robust04 Fold 1 (Title) proposed by Huston &amp; Croft (2014) and used in numerous works</p>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec-robust-2004/fold2:
+ desc: '
+<p>Robust04 Fold 2 (Title) proposed by Huston &amp; Croft (2014) and used in numerous works</p>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec-robust-2004/fold3:
+ desc: '
+<p>Robust04 Fold 3 (Title) proposed by Huston &amp; Croft (2014) and used in numerous works</p>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec-robust-2004/fold4:
+ desc: '
+<p>Robust04 Fold 4 (Title) proposed by Huston &amp; Croft (2014) and used in numerous works</p>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec-robust-2004/fold5:
+ desc: '
+<p>Robust04 Fold 5 (Title) proposed by Huston &amp; Croft (2014) and used in numerous works</p>'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees2004Robust', 'Huston2014ACO']
+
+nocr/trec7:
+ desc: '
+<p>
+The TREC 7 Adhoc Retrieval track.
+</p>
+<ul>
+<li><a href="https://trec.nist.gov/pubs/trec7/papers/overview_7.pdf.gz">Task Overview Paper</a></li>
+</ul>
+'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees1998Trec7']
+
+nocr/trec8:
+ desc: '
+<p>
+The TREC 8 Adhoc Retrieval track.
+</p>
+<ul>
+<li><a href="https://trec.nist.gov/pubs/trec8/papers/overview_8.pdf">Task Overview Paper</a></li>
+</ul>
+'
+ docs_instructions: *inst
+ bibtex_ids: ['Voorhees1996Disks45', 'Voorhees1999Trec8']

ir_datasets/etc/downloads.json

@@ -822,6 +822,55 @@
  }
 },
 
+"disks45": {
+ "docs": {
+ "instructions": "The TREC Robust document collection is from TREC disks 4 and 5. Due to the copyrighted nature of the documents, this collection is for research use only, which requires agreements to be filed with NIST. See details here: <https://trec.nist.gov/data/cd45/index.html>.\nMore details about the procedure can be found here: <https://ir-datasets.com/trec-robust04.html#DataAccess>.\nOnce completed, place the uncompressed source here: {path}\nThis should contain directories like NEWS_data/FBIS, NEWS_data/FR94, etc.",
+ "cache_path": "corpus"
+ },
+ "robust04-queries": {
+ "url": "https://trec.nist.gov/data/robust/04.testset.gz",
+ "irds_mirror": true,
+ "size_hint": 34293,
+ "expected_md5": "5eac3d774a2f87da61c08a94f945beff",
+ "cache_path": "04.testset.gz"
+ },
+ "robust04-qrels": {
+ "url": "https://trec.nist.gov/data/robust/qrels.robust2004.txt",
+ "irds_mirror": true,
+ "size_hint": 6543541,
+ "expected_md5": "123c2a0ba2ec31178cb1050995dcfdfa",
+ "cache_path": "qrels.robust2004.txt"
+ },
+ "trec8-qrels": {
+ "url": "https://trec.nist.gov/data/qrels_eng/qrels.trec8.adhoc.parts1-5.tar.gz",
+ "irds_mirror": true,
+ "size_hint": 325935,
+ "expected_md5": "ce1cfa80b29746d2a5eeddab268d4f6a",
+ "cache_path": "qrels.trec8.adhoc.parts1-5.tar.gz"
+ },
+ "trec8-queries": {
+ "url": "https://trec.nist.gov/data/topics_eng/topics.401-450.gz",
+ "irds_mirror": true,
+ "size_hint": 6946,
+ "expected_md5": "daaafb700eed76f61a6e9e4b0dcc40c8",
+ "cache_path": "topics.401-450.gz"
+ },
+ "trec7-qrels": {
+ "url": "https://trec.nist.gov/data/qrels_eng/qrels.trec7.adhoc.parts1-5.tar.gz",
+ "irds_mirror": true,
+ "size_hint": 307120,
+ "expected_md5": "43def30d4f4b33a830ae67e3dce19023",
+ "cache_path": "qrels.trec7.adhoc.parts1-5.tar.gz"
+ },
+ "trec7-queries": {
+ "url": "https://trec.nist.gov/data/topics_eng/topics.351-400.gz",
+ "irds_mirror": true,
+ "size_hint": 7400,
+ "expected_md5": "fdee3f7e37e173fd6fcdc00fbe1fc671",
+ "cache_path": "topics.351-400.gz"
+ }
+},
+
 "dpr-w100": {
  "docs": {
  "url": "https://dl.fbaipublicfiles.com/dpr/wikipedia_split/psgs_w100.tsv.gz",

ir_datasets/etc/metadata.json

@@ -151,6 +151,16 @@
  "cord19/trec-covid/round4": {"docs": {"count": 158274, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}, "queries": {"count": 45}, "qrels": {"count": 13262, "fields": {"relevance": {"counts_by_value": {"1": 2279, "0": 7438, "2": 3545}}}}},
  "cord19/trec-covid/round5": {"docs": {"_ref": "cord19"}, "queries": {"_ref": "cord19/trec-covid"}, "qrels": {"count": 23151, "fields": {"relevance": {"counts_by_value": {"2": 6677, "1": 4233, "0": 12239, "-1": 2}}}}},
  "cranfield": {"docs": {"count": 1400, "fields": {"doc_id": {"max_len": 4, "common_prefix": ""}}}, "queries": {"count": 225}, "qrels": {"count": 1837, "fields": {"relevance": {"counts_by_value": {"2": 387, "3": 734, "4": 363, "-1": 225, "1": 128}}}}},
+ "disks45": {},
+ "disks45/nocr": {"docs": {"count": 528155, "fields": {"doc_id": {"max_len": 16, "common_prefix": ""}}}},
+ "disks45/nocr/trec-robust-2004": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 250}, "qrels": {"count": 311410, "fields": {"relevance": {"counts_by_value": {"1": 16381, "0": 293998, "2": 1031}}}}},
+ "disks45/nocr/trec-robust-2004/fold1": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 62789, "fields": {"relevance": {"counts_by_value": {"0": 59765, "1": 2795, "2": 229}}}}},
+ "disks45/nocr/trec-robust-2004/fold2": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 63917, "fields": {"relevance": {"counts_by_value": {"1": 3334, "0": 60246, "2": 337}}}}},
+ "disks45/nocr/trec-robust-2004/fold3": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 62901, "fields": {"relevance": {"counts_by_value": {"0": 58859, "1": 3877, "2": 165}}}}},
+ "disks45/nocr/trec-robust-2004/fold4": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 57962, "fields": {"relevance": {"counts_by_value": {"0": 55103, "1": 2707, "2": 152}}}}},
+ "disks45/nocr/trec-robust-2004/fold5": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 63841, "fields": {"relevance": {"counts_by_value": {"0": 60025, "1": 3668, "2": 148}}}}},
+ "disks45/nocr/trec7": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 80345, "fields": {"relevance": {"counts_by_value": {"0": 75671, "1": 4674}}}}},
+ "disks45/nocr/trec8": {"docs": {"_ref": "disks45/nocr"}, "queries": {"count": 50}, "qrels": {"count": 86830, "fields": {"relevance": {"counts_by_value": {"0": 82102, "1": 4728}}}}},
  "dpr-w100": {"docs": {"count": 21015324, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}},
  "dpr-w100/natural-questions/dev": {"docs": {"_ref": "dpr-w100"}, "queries": {"count": 6515}, "qrels": {"count": 979893, "fields": {"relevance": {"counts_by_value": {"2": 6515, "1": 44736, "0": 602894, "-1": 325748}}}}},
  "dpr-w100/natural-questions/train": {"docs": {"_ref": "dpr-w100"}, "queries": {"count": 58880}, "qrels": {"count": 8856662, "fields": {"relevance": {"counts_by_value": {"2": 58880, "1": 405729, "0": 5448064, "-1": 2943989}}}}},