trec-dl-2022 topics and scoreddocs (#200)

Author: seanmacavaney

ir_datasets/datasets/msmarco_document_v2.py

@@ -197,6 +197,11 @@ def _init():
  FilteredScoredDocs(subsets['trec-dl-2021'].scoreddocs_handler(), dl21_judged),
  subsets['trec-dl-2021'],
  )
+ subsets['trec-dl-2022'] = Dataset(
+ collection,
+ TsvQueries(dlc['trec-dl-2022/queries'], namespace='msmarco', lang='en'),
+ TrecScoredDocs(GzipExtract(dlc['trec-dl-2022/scoreddocs'])),
+ )
 
  subsets['anchor-text'] = Dataset(
  MsMarcoV2AnchorTextDocs(

ir_datasets/datasets/msmarco_passage_v2.py

@@ -272,6 +272,11 @@ def _init():
  FilteredScoredDocs(subsets['trec-dl-2021'].scoreddocs_handler(), dl21_judged),
  subsets['trec-dl-2021'],
  )
+ subsets['trec-dl-2022'] = Dataset(
+ collection,
+ TsvQueries(dlc['trec-dl-2022/queries'], namespace='msmarco', lang='en'),
+ TrecScoredDocs(GzipExtract(dlc['trec-dl-2022/scoreddocs'])),
+ )
 
  ir_datasets.registry.register(NAME, Dataset(collection, documentation("_")))
  for s in sorted(subsets):

ir_datasets/docs/msmarco-document-v2.yaml

@@ -106,6 +106,13 @@ Note that at this time, this is only available to those with TREC active partici
 '
  official_measures: ['AP@100', 'nDCG@10', 'P@10', 'RR(rel=2)']
 
+trec-dl-2022:
+ desc: '
+<p>
+Official topics for the TREC Deep Learning (DL) 2022 shared task.
+</p>
+'
+
 anchor-text:
  pretty_name: "Anchor Text for version 2 of MS Marco"
  desc: '

ir_datasets/docs/msmarco-passage-v2.yaml

@@ -77,3 +77,10 @@ with qrels.
 </p>
 '
  official_measures: ['AP@100', 'nDCG@10', 'P(rel=2)@10', 'RR(rel=2)']
+
+trec-dl-2022:
+ desc: '
+<p>
+Official topics for the TREC Deep Learning (DL) 2022 shared task.
+</p>
+'

ir_datasets/etc/downloads.json

@@ -2735,6 +2735,18 @@
  "size_hint": 751699569,
  "expected_md5": "8b96dbaf4efcae08e0ee307e03f3434d",
  "cache_path": "anchor-text-separate-v2.jsonl.gz"
+ },
+ "trec-dl-2022/queries": {
+ "url": "https://msmarco.blob.core.windows.net/msmarcoranking/2022_queries.tsv",
+ "size_hint": 21508,
+ "expected_md5": "f1bfd53d80e81e58207ce557fd2211a0",
+ "cache_path": "trec-dl-2022/queries.tsv"
+ },
+ "trec-dl-2022/scoreddocs": {
+ "url": "https://msmarco.blob.core.windows.net/msmarcoranking/2022_document_top100.txt.gz",
+ "size_hint": 642721,
+ "expected_md5": "93f70329ce1b9ce913a5f87008736ff2",
+ "cache_path": "trec-dl-2022/top100.txt.gz"
  }
 },
 
@@ -2931,6 +2943,18 @@
  "size_hint": 433887,
  "expected_md5": "c5b76ec95b589732edc9040302e22a2b",
  "cache_path": "trec-dl-2021/qrels"
+ },
+ "trec-dl-2022/queries": {
+ "url": "https://msmarco.blob.core.windows.net/msmarcoranking/2022_queries.tsv",
+ "size_hint": 21508,
+ "expected_md5": "f1bfd53d80e81e58207ce557fd2211a0",
+ "cache_path": "trec-dl-2022/queries.tsv"
+ },
+ "trec-dl-2022/scoreddocs": {
+ "url": "https://msmarco.blob.core.windows.net/msmarcoranking/2022_passage_top100.txt.gz",
+ "size_hint": 630095,
+ "expected_md5": "36004dfad64826167aeecddff1d490a6",
+ "cache_path": "trec-dl-2022/top100.txt.gz"
  }
 },
 

ir_datasets/etc/metadata.json

@@ -417,6 +417,7 @@
  "msmarco-document-v2/trec-dl-2020/judged": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 45}, "qrels": {"_ref": "msmarco-document-v2/trec-dl-2020"}},
  "msmarco-document-v2/trec-dl-2021": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 477}, "scoreddocs": {"count": 47700}, "qrels": {"count": 13058, "fields": {"relevance": {"counts_by_value": {"2": 2769, "0": 4855, "3": 1256, "1": 4178}}}}},
  "msmarco-document-v2/trec-dl-2021/judged": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 57}, "qrels": {"_ref": "msmarco-document-v2/trec-dl-2021"}, "scoreddocs": {"count": 5700}},
+ "msmarco-document-v2/trec-dl-2022": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 500}, "scoreddocs": {"count": 50000}},
  "msmarco-document/anchor-text": {"docs": {"count": 1703834, "fields": {"doc_id": {"max_len": 8, "common_prefix": "D"}}}},
  "msmarco-document/dev": {"docs": {"_ref": "msmarco-document"}, "queries": {"count": 5193}, "qrels": {"count": 5193, "fields": {"relevance": {"counts_by_value": {"1": 5193}}}}, "scoreddocs": {"count": 519300}},
  "msmarco-document/eval": {"docs": {"_ref": "msmarco-document"}, "queries": {"count": 5793}, "scoreddocs": {"count": 579300}},
@@ -439,6 +440,7 @@
  "msmarco-passage-v2/train": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 277144}, "qrels": {"count": 284212, "fields": {"relevance": {"counts_by_value": {"1": 284212}}}}, "scoreddocs": {"count": 27713673}},
  "msmarco-passage-v2/trec-dl-2021": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 477}, "scoreddocs": {"count": 47700}, "qrels": {"count": 10828, "fields": {"relevance": {"counts_by_value": {"0": 4338, "3": 1086, "1": 3063, "2": 2341}}}}},
  "msmarco-passage-v2/trec-dl-2021/judged": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 53}, "qrels": {"_ref": "msmarco-passage-v2/trec-dl-2021"}, "scoreddocs": {"count": 5300}},
+ "msmarco-passage-v2/trec-dl-2022": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 500}, "scoreddocs": {"count": 50000}},
  "msmarco-passage/dev": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 101093}, "qrels": {"count": 59273, "fields": {"relevance": {"counts_by_value": {"1": 59273}}}}},
  "msmarco-passage/dev/judged": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 55578}, "qrels": {"_ref": "msmarco-passage/dev"}},
  "msmarco-passage/dev/small": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 6980}, "qrels": {"count": 7437, "fields": {"relevance": {"counts_by_value": {"1": 7437}}}}, "scoreddocs": {"count": 6668967}},

test/integration/msmarco_document_v2.py

@@ -59,6 +59,11 @@ def test_queries(self):
  9: GenericQuery('1104447', 'which kind of continental boundary is formed where two plates move horizontally past one another?'),
  56: GenericQuery('1040198', 'who is the final arbiter of florida law in instances where there is no federal authority?'),
  })
+ self._test_queries('msmarco-document-v2/trec-dl-2022', count=500, items={
+ 0: GenericQuery('588', '1099 b cost basis i sell specific shares'),
+ 9: GenericQuery('77640', "can you get a master's degree in tefl"),
+ 499: GenericQuery('2056473', 'is a dairy farm considered as an agriculture'),
+ })
 
  def test_qrels(self):
  self._test_qrels('msmarco-document-v2/train', count=331956, items={
@@ -128,6 +133,11 @@ def test_scoreddocs(self):
  9: GenericScoredDoc('2082', 'msmarco_doc_01_1320056135', 14.554399),
  47699: GenericScoredDoc('1136769', 'msmarco_doc_57_1870160943', 15.2451),
  })
+ self._test_scoreddocs('msmarco-document-v2/trec-dl-2022', count=50000, items={
+ 0: GenericScoredDoc('588', 'msmarco_doc_01_1675156368', 16.856501),
+ 9: GenericScoredDoc('588', 'msmarco_doc_44_970138133', 16.118099),
+ 49999: GenericScoredDoc('2056473', 'msmarco_doc_58_500974264', 10.868498),
+ })
 
  def test_anchor_text(self):
  self._test_docs("msmarco-document-v2/anchor-text", count=4821244, items={

test/integration/msmarco_passage_v2.py

@@ -39,6 +39,11 @@ def test_queries(self):
  9: GenericQuery('1107704', 'what was the main benefit of a single european currency?'),
  52: GenericQuery('1040198', 'who is the final arbiter of florida law in instances where there is no federal authority?'),
  })
+ self._test_queries('msmarco-passage-v2/trec-dl-2022', count=500, items={
+ 0: GenericQuery('588', '1099 b cost basis i sell specific shares'),
+ 9: GenericQuery('77640', "can you get a master's degree in tefl"),
+ 499: GenericQuery('2056473', 'is a dairy farm considered as an agriculture'),
+ })
 
  def test_qrels(self):
  self._test_qrels('msmarco-passage-v2/train', count=284212, items={
@@ -88,6 +93,11 @@ def test_scoreddocs(self):
  9: GenericScoredDoc('2082', 'msmarco_passage_30_709623997', 17.350901),
  47699: GenericScoredDoc('1136769', 'msmarco_passage_06_68704200', 14.8941),
  })
+ self._test_scoreddocs('msmarco-passage-v2/trec-dl-2022', count=50000, items={
+ 0: GenericScoredDoc('588', 'msmarco_passage_30_337959223', 18.762699),
+ 9: GenericScoredDoc('588', 'msmarco_passage_10_355039123', 17.7628),
+ 49999: GenericScoredDoc('2056473', 'msmarco_passage_17_225374709', 12.147499),
+ })
 
 
 if __name__ == '__main__':