promptsource/seqio_tasks/dataset_subset_template.csv

comment,do_eval,skip_train,dataset_subset_template,nontrivial_choices_given,nontrivial_choices_hidden,trivial_choices_given,trivial_choices_hidden,generative_non_true_task,generative_non_true_implausible,generative_true_task,negated_answers,counting,non_true_task_other,awkward_phrasing,ungrammatical,template_bug,long_distance,no_sep_2_sentences,verbose,answer_span_indices,non_natural_language
,,,adversarial_qa_dbert_adversarial_qa_dbert_1,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_10,,,,,,,,,,,,,,,,,True,True
,,,adversarial_qa_dbert_adversarial_qa_dbert_2,,,,,,,,,,,,,,True,,,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_3,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_4,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_5,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_6,,,,,,,,,,,,,,,,True,,
,,,adversarial_qa_dbert_adversarial_qa_dbert_7,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_dbert_adversarial_qa_dbert_8,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_dbert_adversarial_qa_dbert_9,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_1,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_10,,,,,,,,,,,,,,,,,True,True
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_2,,,,,,,,,,,,,,True,,,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_3,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_4,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_5,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_6,,,,,,,,,,,,,,,,True,,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_7,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_8,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_dbidaf_adversarial_qa_dbidaf_9,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_droberta_adversarial_qa_droberta_1,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_10,,,,,,,,,,,,,,,,,True,True
,,,adversarial_qa_droberta_adversarial_qa_droberta_2,,,,,,,,,,,,,,True,,,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_3,,,,,,,,,,,,,,,,,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_4,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_5,,,,,True,,,,,,,,,,,,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_6,,,,,,,,,,,,,,,,True,,
,,,adversarial_qa_droberta_adversarial_qa_droberta_7,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_droberta_adversarial_qa_droberta_8,,,,,,,,,,,,,,,,,True,
,,,adversarial_qa_droberta_adversarial_qa_droberta_9,,,,,,,,,,,,,,,,,True,
,,,ag_news_classify,,True,,,,,,,,,,,,,,,,
,,,ag_news_classify_with_choices,True,,,,,,,,,,,,,,,,,
,,,ag_news_recommend,True,,,,,,,,,,,,,,,,,
,,,ag_news_which_section,,True,,,,,,,,,,,,,,,,
,,,ag_news_which_section_choices,True,,,,,,,,,,,,,,,,,
,,,amazon_polarity_Template_1,,,True,,,,,,,,,,,,,,,
,,,amazon_polarity_Template_2,,,,True,,,,,,,,,,True,,,,
,,,amazon_polarity_Template_3,,,,True,,,,,,,,,,,,,,
,,,amazon_polarity_Template_4,,,,True,,,,,,,,,,True,,,,
,,,amazon_polarity_Template_5,,,True,,,,,,,,,,,,,,,
,,,amazon_polarity_Template_6,,,True,,,,,,,,,,,True,,,,
,True,True,anli_GPT_3_style_r1,True,,,,,,,,,,,,,,,,,
,True,True,anli_based_on_the_previous_passage_r1,True,,,,,,,,,,,,,,,,,
,True,True,anli_does_S1_contradict_S2__r1,,,,,,,,True,,True,,,,,,,,
,True,True,anli_does_S1_entail_S2__r1,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_does_it_follow_that__r1,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_it_must_be_true_that__r1,True,,,,,,,,,,,,,,,,,
,True,True,anli_GPT_3_style_r2,True,,,,,,,,,,,,,,,,,
,True,True,anli_based_on_the_previous_passage_r2,True,,,,,,,,,,,,,,,,,
,True,True,anli_does_S1_contradict_S2__r2,,,,,,,,True,,True,,,,,,,,
,True,True,anli_does_S1_entail_S2__r2,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_does_it_follow_that__r2,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_it_must_be_true_that__r2,True,,,,,,,,,,,,,,,,,
,True,True,anli_GPT_3_style_r3,True,,,,,,,,,,,,,,,,,
,True,True,anli_based_on_the_previous_passage_r3,True,,,,,,,,,,,,,,,,,
,True,True,anli_does_S1_contradict_S2__r3,,,,,,,,True,,True,,,,,,,,
,True,True,anli_does_S1_entail_S2__r3,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_does_it_follow_that__r3,True,,,,,,,,,,,,,,,,,
,True,True,anli_given_it_must_be_true_that__r3,True,,,,,,,,,,,,,,,,,
,,,app_reviews_categorize_rating_using_review,,True,,,,,,,,,,,,,,,,
,,,app_reviews_convert_to_rating,True,,,,,,,,,,,,,,,,,
,,,app_reviews_convert_to_star_rating,,,,,,,,,,True,,,,,,,,
,,,app_reviews_generate_review,,,,,True,True,,,,,,,,,,,,
,,,ai2_arc_ARC_Challenge_answer_qn,,,,,True,True,,,,,,,,,,,,
,,,ai2_arc_ARC_Challenge_false,,,,,,,,True,,,,,,,,,,
,,,ai2_arc_ARC_Challenge_qa_options,True,,,,,,,,,,,,,,,,,
,,,ai2_arc_ARC_Challenge_test,True,,,,,,,,,,,,,,,,,
,,,ai2_arc_ARC_Easy_answer_qn,,,,,True,True,,,,,,,,,,,,
,,,ai2_arc_ARC_Easy_false,,,,,,,,True,,,,,,,,,,
,,,ai2_arc_ARC_Easy_qa_options,True,,,,,,,,,,,,,,,,,
,,,ai2_arc_ARC_Easy_test,True,,,,,,,,,,,,,,,,,
,True,,circa_goldstandard1_judgement,True,,,,,,,,,,True,,,,,,,
,True,,circa_goldstandard2_judgement,True,,,,,,,,,,True,,,,,,,
,,,circa_judgement,,True,,,,,,,,True,True,,,,,,,
,,,circa_possible_qn,,,,,True,,,,,,,,,,,,,
,,,circa_question_declarative,,,,,,,,,,True,,,,,,,,
,,,cnn_dailymail_3.0.0_generate_story,,,,,True,,,,,,,,,,,,,
,,,cnn_dailymail_3.0.0_news_card_view,,,,,,,True,,,,,,,True,,,,
,,,cnn_dailymail_3.0.0_news_stock,,,,,,,True,,,,,,,True,,,,
,,,cnn_dailymail_3.0.0_news_summary,,,,,,,True,,,,,,,True,,True,,
,,,cnn_dailymail_3.0.0_spice_up_story,,,,,True,,,,,,,,,,,,,
,,,codah_codah_answer_no_option,,True,,,,,,,,,,,,,,,,
,,,codah_codah_answer_with_option,True,,,,,,,,,,,,,,,,,
,,,codah_codah_answer_with_option_idx,True,,,,,,,,,,,,,,,,,
,,,codah_codah_answer_with_option_post,True,,,,,,,,,,,,,,,,,
,,,codah_codah_choose_from_list,True,,,,,,,,,,,,,,,,,
,,,codah_codah_finish_from_the_list,True,,,,,,,,,,,,,,,,,
,,,codah_codah_finish_from_the_list_post,True,,,,,,,,,,,,,,,,,
,,,codah_codah_finish_pre,,True,,,,,,,,,,,,,,,,
,,,codah_codah_question_category,,,,,,,,,,True,,,,,,,,
,,,codah_codah_question_category_bis,,,,,,,,,,True,,,,,,,,
,,,common_gen_Example_prompt,,,,,,,True,,,,,,,,,,,
,,,common_gen_Given_concepts,,,,,,,True,,,,,,,,,,,
,,,common_gen_Put_together,,,,,,,True,,,,,,,,,,,
,,,common_gen_choice_in_concept_centric_sentence_generation,,,,,,,True,,,,,,,,,,,
,,,common_gen_sentence_to_concepts,,,,,,,,,,True,,,,,,,,
,,,cos_e_v1.11_description_question_option_id,True,,,,,,,,,,,,,,,,,
,,,cos_e_v1.11_description_question_option_text,True,,,,,,,,,,,,,,,,,
,,,cos_e_v1.11_generate_explanation_given_text,True,,,,,,True,,,,,,True,,,,,
,,,cos_e_v1.11_generate_explanation_no_given_answer,,True,,,,,True,,,,,,,,,,,
,,,cos_e_v1.11_question_description_option_id,True,,,,,,,,,,,,,,,,,
,,,cos_e_v1.11_question_description_option_text,True,,,,,,,,,,,,,,,,,
,,,cos_e_v1.11_question_option_description_id,True,,,,,,,,,,,,,,,,,
,,,cos_e_v1.11_question_option_description_text,True,,,,,,,,,,,,,,,,,
revisit,,,cosmos_qa_context_description_question_answer_id,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_description_question_answer_text,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_description_question_text,,True,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_question_answer_description_id,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_question_answer_description_text,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_question_description_answer_id,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_question_description_answer_text,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_context_question_description_text,,True,,,,,,,,,,,,,,,,
,,,cosmos_qa_description_context_question_answer_id,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_description_context_question_answer_text,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_description_context_question_text,,True,,,,,,,,,,,,,,,,
,,,cosmos_qa_no_prompt_id,True,,,,,,,,,,,,,,,,,
,,,cosmos_qa_no_prompt_text,True,,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_1,,True,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_10,True,,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_3,,True,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_5,,True,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_7,,True,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_8,,True,,,,,,,,,,,,,,,,
,,,dbpedia_14_dbpedia_9,True,,,,,,,,,,,,,,,,,
,,,dream_answer_to_dialogue,,,,,True,,,,,,,,,,,,,
,,,dream_baseline,True,,,,,,,,,,,,,,,,,
,,,dream_conversation,True,,,,,,,,,,,,,,,,,
,,,dream_generate_first_utterance,,,,,True,,,,,,,,,,,,,
,,,dream_generate_last_utterance,,,,,True,,,,,,,,,,,,,
,True,,emo_feeling,True,,,,,,,,,,,,,,,,,
,True,,emo_final_message,True,,,,,,,,,,,,,,,,,
,True,,emo_persons_describe,True,,,,,,,,,,,,,,,True,,
,True,,emo_persons_infer,True,,,,,,,,,,,,,,,,,
,True,,emo_spoke_last,True,,,,,,,,,,,,,,,,,
,,,freebase_qa_inference_chain_prompt,,,,,,,,,,True,,,,,,,,
,,,freebase_qa_inference_chain_prompt_context,,,,,,,,,,True,,,,,,,,
,,,freebase_qa_qa_context_1,,,,,,,,,,,,,,,,,,
,,,freebase_qa_qa_context_2,,,,,,,,,,,,,,,,,,
,,,freebase_qa_qa_template_basic,,,,,,,,,,,,,,,,,,
,,,gigaword_Document_,,,,,,,True,,,,,,,,,,,
,,,gigaword_Summarize_this_document_,,,,,,,True,,,,,,,,,,,
,,,gigaword_TLDR,,,,,,,True,,,,,,,,,,,
,,,gigaword_generate_summary_for_this,,,,,,,True,,,,,,,,,,,
,,,gigaword_in_a_nutshell,,,,,,,True,,,,,,,,,,,
,,,gigaword_reverse_writing,,,,,,,,,,True,,,,,,,,
,,,gigaword_reverse_writing_2,,,,,,,True,,,,,,,,,,,
,,,gigaword_summarize_,,,,,,,True,,,,,,,,,,,
,,,gigaword_write_one_sentence,,,,,,,True,,,,,,,,,,,
,True,True,glue_cola_Following_sentence_acceptable,True,,,,,,,,,,,,,,,,,
,True,True,glue_cola_Make_sense_yes_no,,,True,,,,,,,,,,,,,,,
,True,True,glue_cola_Previous_sentence_acceptable,,,,True,,,,,,,,,,,,,,
,True,True,glue_cola_editing,,,True,,,,,,,,,,,,,,,
,True,True,glue_cola_jinja_example,,,,True,,,,,,,,,,,,,,
,True,,glue_mrpc_equivalent,True,,,,,,,,,,,,,,True,,,
,True,,glue_mrpc_paraphrase,,,,True,,,,,,,,,,,,,,
,True,,glue_mrpc_replace,,,,True,,,,,,,,,,,,,,
,True,,glue_mrpc_same_thing,,,,True,,,,,,,,,,,True,,,
,True,,glue_mrpc_want_to_know,,,,True,,,,,,,,,,,True,,,
,,,glue_qqp_answer,,,,True,,,,,,,,,,,,,,
,,,glue_qqp_duplicate,,,,True,,,,,,,,,,,,,,
,,,glue_qqp_duplicate_or_not,True,,,,,,,,,,,,,,,,,
,,,glue_qqp_quora,,,,True,,,,,,,,,,,,True,,
,,,glue_qqp_same_thing,,,,True,,,,,,,,,,,,,,
,,,glue_sst2_following_positive_negative,True,,,,,,,,,,,,,,,,,
,,,glue_sst2_happy_or_mad,True,,,,,,,,,,,,,,,,,
,,,glue_sst2_positive_negative_after,True,,,,,,,,,,,,,,,,,
,,,glue_sst2_review,True,,,,,,,,,,,,,,,,,
,,,glue_sst2_said,True,,,,,,,,,,,,,,,,,
,,True,glue_stsb_examples,,,,,,,,,,,,,,,,,,
,,True,glue_stsb_rank,,,,,,,,,,,,,,,,,,
,,True,glue_stsb_rate,,,,,,,,,,,,,,,,,,
,,True,glue_stsb_score,,,,,,,,,,,,,,,,,,
,,True,glue_stsb_similarity,,,,,,,,,,,,,,,,,,
,True,True,hans_GPT_3_style,True,,,,,,,,,,,,,,,,,
,True,True,hans_Suppose_Can_we_infer_that_,,,,True,,,,,,,,,,,,,,
,True,True,hans_based_on_the_previous_passage,,,,True,,,,,,,,,,,,,,
,True,True,hans_does_S1_entail_S2_,,,True,,,,,,,,,,,,,,,
,True,True,hans_given_does_it_follow_that_,,,True,,,,,,,,,,,,,,,
,True,True,hans__does_the_previous_passage_support_the_claim_that,,,,True,,,,,,,,,,,,,,
,,,hellaswag_YesNo_0,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_1,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_2,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_3,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_reversed_0,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_reversed_1,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_reversed_2,,,True,,,,,,,,,,,,,,,
,,,hellaswag_YesNo_reversed_3,,,True,,,,,,,,,,,,,,,
,,,hellaswag_complete_first_then,True,,,,,,,,,,,,,,,,,
,,,hellaswag_first_then,True,,,,,,,,,,,,,,,,,
,,,hellaswag_how_ends,True,,,,,,,,,,,,,,,,,
,,,hellaswag_if_begins_how_continues,True,,,,,,,,,,,,,,,,,
,,,hellaswag_which_ending,True,,,,,,,,,,,,,,,,,
,,,imdb_imdb_1,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_2,,True,,,,,,True,,,,,,,,,,
,,,imdb_imdb_3,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_4,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_5,,True,,,,,,,,,,,,True,,,,
,,,imdb_imdb_6,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_7,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_8,,True,,,,,,,,,,,,,,,,
,,,imdb_imdb_9,,,,True,,,,,,,,,,,,,,
,True,,mc_taco_mc_taco_1,,,,True,,,,,,,,,,,,,,
,,,mc_taco_mc_taco_2,,,,,,,,,,True,,,,,,,,
,True,,mc_taco_mc_taco_3,,,True,,,,,,,,,,,True,,,,
,,,mc_taco_mc_taco_4,True,,,,,,,,,True,,,,,,,,
,,,mc_taco_mc_taco_5,,,,,True,,,,,,,,,,,,,
,,,mc_taco_mc_taco_6,,True,,,,,,,,,,,,,,,,
,True,True,nq_open_context_self_description,,,,,,,,,,,,,,,,,,
,,True,nq_open_guess_question,,,,,True,,,,,,,,,,,,,
,True,True,nq_open_question_answer,,,,,,,,,,,,,,,,,,
,True,True,nq_open_question_with_instruction,,,,,,,,,,,,,,,,,,
,,,onestop_english_ara_context,True,,,,,,,,,,,,,,,,,
,,,onestop_english_assess,True,,,,,,,,,,,,,True,,,,
,,,onestop_english_ats,True,,,,,,,,,,,,,,,,,
,,,onestop_english_esl_context,True,,,,,,,,,,,,,True,,,,
,,,onestop_english_esl_variation,True,,,,,,,,,,,,,True,,,,
,True,,openbookqa_main_choices,True,,,,,,,,,,,,,,,,,
,True,,openbookqa_main_choose_an_answer_with_options,True,,,,,,,,,,,,,,,,,
,True,,openbookqa_main_only_options,True,,,,,,,,,,,,,,,,,
,True,,openbookqa_main_pick_answer_with_options,True,,,,,,,,,,,,,,,,,
,True,,openbookqa_main_pick_using_id,True,,,,,,,,,,,,,,,,,
,True,,openbookqa_main_which_correct,True,,,,,,,,,,,,,,,,,
,,True,openbookqa_main_which_correct_inverse,True,,,,,,,,,,,,True,,,,,
,,,paws_labeled_final_Concatenation,,,True,,,,,,,,,,True,,,,,
,,,paws_labeled_final_Concatenation_no_label,,,,True,,,,,,,,,True,,,,,
,,,paws_labeled_final_Meaning,,,True,,,,,,,,,,True,,,,,
,,,paws_labeled_final_Meaning_no_label,,,,True,,,,,,,,,True,,,,,
,,,paws_labeled_final_PAWS_ANLI_GPT3,True,,,,,,,,,True,,,,,,,,
,,,paws_labeled_final_PAWS_ANLI_GPT3_no_label,,True,,,,,,,,True,,,,,,,,
,,,piqa_Correct_the_solution,,,,,True,,,,,,,,,,,,,
,,,piqa_Correct_the_solution_if_false_from_sol_1,,,,,True,,,,,,,,,,,,,
,,,piqa_Correct_the_solution_if_false_from_sol_2,,,,,True,,,,,,,,,,,,,
should use jinja choice,,,piqa_Does_this_solution_make_sense_sol1,,,,True,,,,,,,,,,,,,,
,,,piqa_Does_this_solution_make_sense_sol2,,,,True,,,,,,,,,,,,,,
,,,piqa_Generate_a_similar_but_wrong_solution,,,,,True,,,,,,,,,,,,,
,,,piqa_choose_the_most_appropriate_solution,True,,,,,,,,,,,,,,,,,
duplicate of above,,True,piqa_choose_the_most_appropriate_solution_reorder_solution,True,,,,,,,,,,,,,,,,,
,,,piqa_no_prompt_needed,,,,,True,,,,,,,,,,,,,
,,,qa_srl_aq,,,,,True,True,,,,,,,,,,,,
,,,qa_srl_context_answer,,,,,True,,,,,,,,,,,,,
,,,qa_srl_context_qn,,,,,True,,,,,,,,,,,,,
,,,qa_srl_predicate,,,,,,,,,,True,,,,,,,,
need non-naive metric,True,,qa_srl_qa,,,,,,,,,,,,,,,,,,
,,,qasc_is_correct_0,,,,True,,,,,,,,,,,,,,
,,,qasc_is_correct_1,,,,True,,,,,,,,,,,,,,
,,,qasc_qu_combined,True,,,,,,,,,,,,,,,,,
,,,qasc_sep_combined_can_tell,True,,,,,,,,,,,,,,,,,
,,,qasc_sep_qu,True,,,,,,,,,,,,,,,,,
,,,quail_context_description_question_answer_id,True,,,,,,,,,,,,,,,,,
,,,quail_context_description_question_answer_text,True,,,,,,,,,,,,,,,,,
,,,quail_context_description_question_text,,True,,,,,,,,,,,,,,,,
,,,quail_context_question_answer_description_id,True,,,,,,,,,,,,,,,,,
,,,quail_context_question_answer_description_text,True,,,,,,,,,,,,,,,,,
,,,quail_context_question_description_answer_id,True,,,,,,,,,,,,,,,,,
,,,quail_context_question_description_answer_text,True,,,,,,,,,,,,,,,,,
,,,quail_context_question_description_text,True,,,,,,,,,,,,,,,,,
,,,quail_description_context_question_answer_id,,True,,,,,,,,,,,,,,,,
,,,quail_description_context_question_answer_text,True,,,,,,,,,,,,,,,,,
,,,quail_description_context_question_text,,True,,,,,,,,,,,,,,,,
,,,quail_no_prompt_id,True,,,,,,,,,,,,,,,,,
,,,quail_no_prompt_text,True,,,,,,,,,,,,,,,,,
,,,quartz_para_question_1,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,quartz_para_question_1_reverse,True,,,,,,,,,,,,,,,,,
,,,quartz_para_question_2,True,,,,,,,,,,,,,,,,,
,,,quartz_para_question_3_choices,True,,,,,,,,,,,,,,,,,
,,,quartz_para_question_4_choices,True,,,,,,,,,,,,,,,,,
,,,quartz_para_question_plain,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,quartz_para_question_plain_reverse,True,,,,,,,,,,,,,,,,,
,,,quartz_question_para_1,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,quartz_question_para_1_reverse,True,,,,,,,,,,,,,,,,,
,,,quartz_question_para_2,True,,,,,,,,,,,,,,,,,
,,,quartz_question_para_3,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,quartz_question_para_3_reverse,True,,,,,,,,,,,,,,,,,
,,,quoref_Template_1,,,,,,,,,,,,,,,,,,
,,,quoref_Template_2,,,,,,,,,,,,,,True,,,,
,,,quoref_Template_3,,,,,True,,,,,,True,,,,,,,
,,,quoref_Template_4,,,,,,,,,,True,,,,,,,True,
,,,quoref_Template_5,,,,,,,,,,True,,,,,,,,
,,,race_high_Read_the_article_and_answer_the_question_no_option_,,True,,,,,,,,,,,,,,,,
,True,,race_high_Read_the_article_and_select_the_best_answer,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,race_high_Read_the_article_and_select_the_best_answer2,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,race_high_Read_the_article_and_select_the_best_answer3,True,,,,,,,,,,,,,,,,,
,,,race_high_Write_a_multi_choice_question_for_the_following_article,,,,,True,,,,,,,,,,,,,
,,,race_high_Write_a_multi_choice_question_for_the_following_article_2,,,,,True,,,,,,,,,,,,,
,,,race_middle_Read_the_article_and_answer_the_question_no_option_,,True,,,,,,,,,,,,,,,,
,True,,race_middle_Read_the_article_and_select_the_best_answer,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,race_middle_Read_the_article_and_select_the_best_answer2,True,,,,,,,,,,,,,,,,,
near duplicate of the above,,True,race_middle_Read_the_article_and_select_the_best_answer3,True,,,,,,,,,,,,,,,,,
,,,race_middle_Write_a_multi_choice_question_for_the_following_article,,,,,True,,,,,,,,,,,,,
,,,race_middle_Write_a_multi_choice_question_for_the_following_article_2,,,,,True,,,,,,,,,,,,,
,,,ropes_funky_prompt,True,,,,,,,,,,,,,,,,,
,,,ropes_plain,True,,,,,,,,,,,,,,,,,
,,,ropes_plain_bottom_hint,True,,,,,,,,,,,,,True,,,,
,,,ropes_plain_no_background,True,,,,,,,,,True,,,,,,,,
,,,ropes_prompt_beginning,True,,,,,,,,,,,,,,,,,
,,,ropes_prompt_bottom_hint_beginning,True,,,,,,,,,,,,,,,,,
,,,ropes_prompt_bottom_no_hint,True,,,,,,,,,True,,,,,,,,
,,,ropes_prompt_mix,True,,,,,,,,,,,,,True,,,,
,,,rotten_tomatoes_rt_1,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_10,True,,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_2,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_3,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_4,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_5,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_6,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_7,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_8,,True,,,,,,,,,,,,,,,,
,,,rotten_tomatoes_rt_9,,,,True,,,,,,,,,,,,,,
,,,sciq_Template_0,,True,,,,,,,,,,,True,,,,,
,,,sciq_Template_1,,True,,,,,,,,,,,True,,,,,
,True,,social_i_qa_social_i_qa1,True,,,,,,,,,,,,,,,,,
,,,social_i_qa_social_i_qa2,,True,,,,,,,,,,,,,,,,
select answer by ordinal word,True,,social_i_qa_social_i_qa3,True,,,,,,,,,,,,,,,,,
,,,social_i_qa_social_i_qa4,,,,,True,,,,,,,,,,,,,
4-way to binary classification,,,social_i_qa_social_i_qa5,,,,True,,,,,,,,,,,,,,
,,,squad_v2_Jeopardy_with_Context,,,,,True,,,,,,,,,,,,,
,,,squad_v2_Jeopardy_without_Context,,,,,True,,,,,True,,,,,,,,
,,,squad_v2_Questions_with_Context,True,,,,,,,,,,,,,,,,,
nicely randomnized prompt phrasing,,,squad_v2_Questions_with_Context_Without_Prompt_Keywords,True,,,,,,,,,,,,,,,,,
,,,squad_v2_Topic_Prediction_Context,,,,,,,,,,True,,,,,,,,
,,,squad_v2_Topic_Prediction_Context_with_randomized_prompt_options,,,,,,,,,,True,,,,,,,,
,,,squad_v2_Topic_Prediction_Context_with_randomized_prompt_options_placed_in_the_end,,,,,,,,,,True,,,,,,,,
,,,squad_v2_Topic_Prediction_Question_and_Answer_Pair,,,,,,,,,,True,,,,,,,,
,,,squad_v2_Trivia,,,,,,,,,,True,,,,,,,,
,True,,super_glue_boolq_GPT_3_Style,,,,True,,,,,,,,,,,,,,
,True,,super_glue_boolq_I_wonder_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_boolq_based_on_the_following_passage,,,,True,,,,,,,,,,,,,,
,True,,super_glue_boolq_based_on_the_previous_passage,,,,True,,,,,,,,,,,,,,
,True,,super_glue_boolq_could_you_tell_me_,,,,True,,,,,,,,,,,,,,
,True,True,super_glue_cb_GPT_3_style,True,,,,,,,,,,,,,,,,,
,True,True,super_glue_cb_based_on_the_previous_passage,True,,,,,,,,,,,,,,,,,
contrapositive,True,True,super_glue_cb_does_S1_contradict_S2_,True,,,,,,,,,True,,,,,,,,
,True,True,super_glue_cb_does_S1_entail_S2_,True,,,,,,,,,,,,,,,,,
,True,True,super_glue_cb_given_does_it_follow_that_,True,,,,,,,,,,,,,,,,,
must/might/may be true,True,True,super_glue_cb_given_it_must_be_true_that_,True,,,,,,,,,,,,,,,,,
,True,,super_glue_copa_C1_or_C2_premise_so_because_,True,,,,,,,,,,,,,,,,,
effect examples,True,,super_glue_copa__As_a_result_C1_or_C2_,True,,,,,,,,,,,,,,,,,
effect examples,True,,super_glue_copa__What_could_happen_next_C1_or_C2_,True,,,,,,,,,,,,,,,,,
cause examples,True,,super_glue_copa__which_may_be_caused_by,True,,,,,,,,,,,,,,,,,
effect examples,True,,super_glue_copa__which_may_cause_C1_or_C2_,True,,,,,,,,,,,,,,,,,
cause examples,True,,super_glue_copa__why_C1_or_C2,True,,,,,,,,,,,,,,,,,
,True,,super_glue_multirc_I_was_going_to_say_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_multirc_Would_it_be_good_to_answer_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_multirc_is_a_correct_answer_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_multirc_is_the_correct_answer_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_multirc_paragraph_question_is_it_,,,,True,,,,,,,,,,,,,,
,True,,super_glue_record_Can_you_figure_out_,,True,,,,,,,,,,,,,,,,
,True,,super_glue_record_In_the_question_above_the_placeholder_stands_for,,True,,,,,,,,,,,,,,,,
,True,,super_glue_record_What_could_the_placeholder_be_,True,,,,,,,,,,,,,,,,,
no difference here?,True,,super_glue_record_Which_one_is_the_placeholder_,True,,,,,,,,,,,,,,,,,
,True,,super_glue_record_the_placeholder_refers_to_,,True,,,,,,,,,,,,,,,,
,True,True,super_glue_rte_GPT_3_style,True,,,,,,,,,,,,,,,,,
,True,True,super_glue_rte_Suppose_Can_we_infer_that_,,,,True,,,,,,,,,,,,,,
,True,True,super_glue_rte_based_on_the_previous_passage,,,,True,,,,,,,,,,,,,,
,True,True,super_glue_rte_does_S1_entail_S2_,,,True,,,,,,,,,,,,,,,
,True,True,super_glue_rte_given_does_it_follow_that_,,,,True,,,,,,,,,,,,,,
,True,True,super_glue_rte__Therefore_we_re_licensed_to_say_that_,,,,True,,,,,,,,,,,,,,
,True,True,super_glue_rte__does_the_previous_passage_support_the_claim_that,,,,True,,,,,,,,,,,,,,
,True,,super_glue_wic_GPT_3_prompt,,,,True,,,,,,,,,,,True,,,
,True,,super_glue_wic_GPT_3_prompt_with_label,,,True,,,,,,,,,,,,True,,,
,True,,super_glue_wic_question_context,,,,True,,,,,,,,,,,True,,,
,True,,super_glue_wic_question_context_meaning,,,,True,,,,,,,,,,,True,,,
,True,,super_glue_wic_question_context_meaning_with_label,,,True,,,,,,,,,,,,True,,,
,True,,super_glue_wic_similar_sense,,,,True,,,,,,,,,,,True,,,
,True,,super_glue_wsc.fixed_Here_p_stands_for_,,,,,,,,,,,,,,,,,,
,True,,super_glue_wsc.fixed_In_the_previous_sentence_the_pronoun_refers_to_,,,,,,,,,,,,,,,,,,
,True,,super_glue_wsc.fixed_Who_is_are_,,,,,,,,,,,,,,,,,,
,True,,super_glue_wsc.fixed_in_the_passage_above_the_pronoun_X_refers_to_,,,,,,,,,,,,,,,,,,
,True,,super_glue_wsc.fixed_passage_what_does_the_pronoun_refer_to_,,,,,,,,,,,,,,,,,,
cast 4-way classification as binary,,,swag_regular_YesNo_0,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_1,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_2,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_3,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_reversed_0,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_reversed_1,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_reversed_2,,,True,,,,,,,,,,,,,,,
,,,swag_regular_YesNo_reversed_3,,,True,,,,,,,,,,,,,,,
,,,swag_regular_complete_first_then,True,,,,,,,,,,,,,,,,,
,,,swag_regular_first_then,True,,,,,,,,,,,,,,,,,
,,,swag_regular_how_ends,True,,,,,,,,,,,,,,,,,
,,,swag_regular_if_begins_how_continues,True,,,,,,,,,,,,,,,,,
,,,swag_regular_which_ending,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_ABBR,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_ABBR_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_DESC,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_DESC_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_ENTY,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_ENTY_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_HUM,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_HUM_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_LOC,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_LOC_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_NUM,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_NUM_context_first,True,,,,,,,,,,,,,,,,,
,,,trec_fine_grained_open,,True,,,,,,,,,,,,,,,,
,,,trec_fine_grained_open_context_first,,True,,,,,,,,,,,,,,,,
answers are not what the questions ask for,,True,trec_gao_et_al_1,,,,,,,,,,,,True,,,,,,
answers are not what the questions ask for,,True,trec_gao_et_al_2,,,,,,,,,,,,True,,,,,,
,,,trec_trec1,True,,,,,,,,,,,,,,,,,
,,,trec_trec2,True,,,,,,,,,,,,,,,,,
,,,trivia_qa_rc_context_self_description,,,,,,,,,,,,,,,,,,
,,,trivia_qa_rc_guess_question,,,,,True,True,,,,True,,,,,,,,
,,,trivia_qa_rc_question_answer,,,,,,,,,,,,,,,,,,
,,,trivia_qa_rc_question_with_instruction,,,,,,,,,,,,,,,,,,
,,,trivia_qa_rc_reading_comprehension_1,,,,,,,,,,True,,,,,,,,
,,,trivia_qa_rc_reading_comprehension_2,,,,,,,,,,True,,,,,,,,
,,,web_questions_count_answers,,,,,,,,,True,,,,,,,,,
,,,web_questions_credible_question,,,,,True,,,,,,,,,,,,,
,,,web_questions_if_answers_what_question,,,,,True,,,,,,,,,,,,,
,,,web_questions_potential_correct_answer,,,,,,,,,,,True,,,,,,,
,,,web_questions_question_answer,,,,,,,,,,,,,,,,,,
,,,web_questions_suggest_question,,,,,True,,,,,,,,,,,,,
,,,wiki_bio_comprehension,,,,,,,,,,True,,,,,,,,
,,,wiki_bio_guess_person,,,,,,,,,,True,,,,,,,,
,,,wiki_bio_key_content,,,,,,,,,,True,,,,,,,,
,,,wiki_bio_what_content,,,,,,,,,,True,,,,,,,,
"should rephrase ""summarize""",,,wiki_bio_who,,,,,,,,,,,,,,,,,,
,,,wiki_hop_original_Choose_Best_Object_Candidate,,,,,,,,,,True,,,,,,,,True
,,,wiki_hop_original_Explain_Relation,,True,,,,,,,,True,,,,,,,,
,,,wiki_hop_original_Generate_Fact_Triple,,,,,,,,,,True,,,,,,,,True
,,,wiki_hop_original_Generate_Object_Answer,,,,,,,,,,True,,,,,,,,True
,,,wiki_hop_original_Generate_Subject_Answer,,,,,,,,,,True,,,,,,,,True
,,,wiki_hop_original_Indirect_Question_about_Birthplace_Citizenship_Place_of_Death,,,,,,,,,,,,,True,,,,,
,,,wiqa_effect_with_label_answer,True,,,,,,,,,,,,,,,,,
,,,wiqa_effect_with_string_answer,True,,,,,,,,,,,,,,,,,
,,,wiqa_impacting_the_process,,,,True,,,,,,,,,,,,,,
,,,wiqa_question_type,,,,,,,,,,True,,,,,,,,
,,,wiqa_remove_first_step,,,,,,,,,,True,,,,,,,,
,,,wiqa_remove_first_step_bis,,,,,,,,,,True,,,,,,,,
,,,wiqa_remove_last_step,,,,,,,,,,True,,,,,,,,
,,,wiqa_remove_last_step_bis,,,,,,,,,,True,,,,,,,,
,True,,xsum_Document_,,,,,,,,,,,,,,,,,,
,True,,xsum_Summarize_this_document_,,,,,,,,,,,,,,,,,,
,True,,xsum_TLDR,,,,,,,,,,,,,,,,,,
,True,,xsum_generate_summary_for_this,,,,,,,,,,,,,,,,,,
,True,,xsum_summarize_,,,,,,,,,,,,,,True,,,,
,True,,xsum_write_one_sentence,,,,,,,,,,,,,,,,,,
,,,yelp_review_full_based_on_that,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_format_rating,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_format_score,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_format_star,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_on_a_scale,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_so_i_would,,True,,,,,,,,,,,,,,,,
,,,yelp_review_full_this_place,,True,,,,,,,,,,,,,,,,