|
| 1 | +dataset: blbooksgenre |
| 2 | +subset: title_genre_classifiction |
| 3 | +templates: |
| 4 | + 0c3e83f4-7f4d-4eca-8f80-6b6bdd8eeedd: !Template |
| 5 | + answer_choices: Fiction ||| Non-fiction |
| 6 | + id: 0c3e83f4-7f4d-4eca-8f80-6b6bdd8eeedd |
| 7 | + jinja: "Given the title: {{title}}, which of the following genres is the book?\n\ |
| 8 | + (a) {{ answer_choices[0] }}\n(b) {{ answer_choices[1] }}\n|||\n {{ answer_choices[label]\ |
| 9 | + \ }}" |
| 10 | + metadata: !TemplateMetadata |
| 11 | + choices_in_prompt: true |
| 12 | + metrics: |
| 13 | + - Accuracy |
| 14 | + - AUC |
| 15 | + original_task: true |
| 16 | + name: multi-choice |
| 17 | + reference: '' |
| 18 | + 5564acb9-c911-4d71-ba4d-add444aaf1e3: !Template |
| 19 | + answer_choices: True ||| False |
| 20 | + id: 5564acb9-c911-4d71-ba4d-add444aaf1e3 |
| 21 | + jinja: "{{title}} is the title of a fictional book, True or False?\nAnswer: \n\ |
| 22 | + |||\n{{ answer_choices[label] }}" |
| 23 | + metadata: !TemplateMetadata |
| 24 | + choices_in_prompt: true |
| 25 | + metrics: |
| 26 | + - Accuracy |
| 27 | + - AUC |
| 28 | + original_task: true |
| 29 | + name: premise_context_first |
| 30 | + reference: '' |
| 31 | + afc18daa-999d-495f-908a-d99477f6f5ac: !Template |
| 32 | + answer_choices: True ||| False |
| 33 | + id: afc18daa-999d-495f-908a-d99477f6f5ac |
| 34 | + jinja: "The following is the title of a fictional book, True or False?\n{{title}}\n\ |
| 35 | + Answer: \n|||\n{{ answer_choices[label] }}" |
| 36 | + metadata: !TemplateMetadata |
| 37 | + choices_in_prompt: true |
| 38 | + metrics: |
| 39 | + - Accuracy |
| 40 | + - AUC |
| 41 | + original_task: true |
| 42 | + name: premise |
| 43 | + reference: '' |
| 44 | + cf4b6ce0-ff87-4c7a-9b9e-ec7c4cf741d8: !Template |
| 45 | + answer_choices: Fiction ||| Non-fiction |
| 46 | + id: cf4b6ce0-ff87-4c7a-9b9e-ec7c4cf741d8 |
| 47 | + jinja: The genre of the book "{{title}}" is ||| {{ answer_choices[label] }} |
| 48 | + metadata: !TemplateMetadata |
| 49 | + choices_in_prompt: false |
| 50 | + metrics: |
| 51 | + - Accuracy |
| 52 | + - AUC |
| 53 | + original_task: true |
| 54 | + name: classify |
| 55 | + reference: '' |
0 commit comments