Upload 17 files

jmteb_config/jmteb.jsonnet

@@ -0,0 +1,22 @@
+// Classification
+(import './tasks/amazon_review_classification.jsonnet') +
+(import './tasks/amazon_counterfactual_classification.jsonnet') +
+(import './tasks/massive_intent_classification.jsonnet') +
+(import './tasks/massive_scenario_classification.jsonnet') +
+// Clustering
+(import './tasks/livedoor_news.jsonnet') +
+(import './tasks/mewsc16.jsonnet') +
+// STS
+(import './tasks/jsts.jsonnet') +
+(import './tasks/jsick.jsonnet') +
+// Pair Classification
+(import './tasks/paws_x_ja.jsonnet') +
+// Retrieval
+(import './tasks/jagovfaqs_22k.jsonnet') +
+(import './tasks/mrtydi.jsonnet') +
+(import './tasks/jaqket.jsonnet') +
+(import './tasks/nlp_journal_title_abs.jsonnet') +
+(import './tasks/nlp_journal_title_intro.jsonnet') +
+(import './tasks/nlp_journal_abs_intro.jsonnet') +
+// Reranking
+(import './tasks/esci.jsonnet')

jmteb_config/tasks/amazon_counterfactual_classification.jsonnet

@@ -0,0 +1,32 @@
+{
+  amazon_counterfactual_classification: {
+    class_path: 'ClassificationEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      train_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'train',
+          name: 'amazon_counterfactual_classification',
+        },
+      },
+      val_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'amazon_counterfactual_classification',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'amazon_counterfactual_classification',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/amazon_review_classification.jsonnet

@@ -0,0 +1,32 @@
+{
+  amazon_review_classification: {
+    class_path: 'ClassificationEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      train_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'train',
+          name: 'amazon_review_classification',
+        },
+      },
+      val_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'amazon_review_classification',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'amazon_review_classification',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/esci.jsonnet

@@ -0,0 +1,33 @@
+{
+  esci: {
+    class_path: 'RerankingEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRerankingQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'esci-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRerankingQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'esci-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRerankingDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'esci-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/jagovfaqs_22k.jsonnet

@@ -0,0 +1,33 @@
+{
+  jagovfaqs_22k: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'jagovfaqs_22k-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'jagovfaqs_22k-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'jagovfaqs_22k-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/jaqket.jsonnet

@@ -0,0 +1,33 @@
+{
+  jaqket: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'jaqket-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'jaqket-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'jaqket-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/jsick.jsonnet

@@ -0,0 +1,25 @@
+{
+  jsick: {
+    class_path: 'STSEvaluator',
+    init_args: {
+      sentence1_prefix: 'クエリ: ',
+      sentence2_prefix: 'クエリ: ',
+      val_dataset: {
+        class_path: 'HfSTSDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'jsick',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfSTSDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'jsick',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/jsts.jsonnet

@@ -0,0 +1,25 @@
+{
+  jsts: {
+    class_path: 'STSEvaluator',
+    init_args: {
+      sentence1_prefix: 'クエリ: ',
+      sentence2_prefix: 'クエリ: ',
+      val_dataset: {
+        class_path: 'HfSTSDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'train',
+          name: 'jsts',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfSTSDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'jsts',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/livedoor_news.jsonnet

@@ -0,0 +1,24 @@
+{
+  livedoor_news: {
+    class_path: 'ClusteringEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      val_dataset: {
+        class_path: 'HfClusteringDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'livedoor_news',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClusteringDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'livedoor_news',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/massive_intent_classification.jsonnet

@@ -0,0 +1,32 @@
+{
+  massive_intent_classification: {
+    class_path: 'ClassificationEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      train_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'train',
+          name: 'massive_intent_classification',
+        },
+      },
+      val_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'massive_intent_classification',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'massive_intent_classification',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/massive_scenario_classification.jsonnet

@@ -0,0 +1,32 @@
+{
+  massive_scenario_classification: {
+    class_path: 'ClassificationEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      train_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'train',
+          name: 'massive_scenario_classification',
+        },
+      },
+      val_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'massive_scenario_classification',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'massive_scenario_classification',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/mewsc16.jsonnet

@@ -0,0 +1,24 @@
+{
+  mewsc16: {
+    class_path: 'ClusteringEvaluator',
+    init_args: {
+      prefix: 'クエリ: ',
+      val_dataset: {
+        class_path: 'HfClusteringDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'mewsc16_ja',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfClusteringDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'mewsc16_ja',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/mrtydi.jsonnet

@@ -0,0 +1,33 @@
+{
+  mrtydi: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'mrtydi-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'mrtydi-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'mrtydi-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/nlp_journal_abs_intro.jsonnet

@@ -0,0 +1,33 @@
+{
+  nlp_journal_abs_intro: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: '文章: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'nlp_journal_abs_intro-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'nlp_journal_abs_intro-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'nlp_journal_abs_intro-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/nlp_journal_title_abs.jsonnet

@@ -0,0 +1,33 @@
+{
+  nlp_journal_title_abs: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'nlp_journal_title_abs-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'nlp_journal_title_abs-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'nlp_journal_title_abs-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/nlp_journal_title_intro.jsonnet

@@ -0,0 +1,33 @@
+{
+  nlp_journal_title_intro: {
+    class_path: 'RetrievalEvaluator',
+    init_args: {
+      doc_prefix: '文章: ',
+      query_prefix: 'クエリ: ',
+      val_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'nlp_journal_title_intro-query',
+        },
+      },
+      test_query_dataset: {
+        class_path: 'HfRetrievalQueryDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'nlp_journal_title_intro-query',
+        },
+      },
+      doc_dataset: {
+        class_path: 'HfRetrievalDocDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'corpus',
+          name: 'nlp_journal_title_intro-corpus',
+        },
+      },
+    },
+  },
+}

jmteb_config/tasks/paws_x_ja.jsonnet

@@ -0,0 +1,25 @@
+{
+  paws_x_ja: {
+    class_path: 'PairClassificationEvaluator',
+    init_args: {
+      sentence1_prefix: 'クエリ: ',
+      sentence2_prefix: 'クエリ: ',
+      val_dataset: {
+        class_path: 'HfPairClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'validation',
+          name: 'paws_x_ja',
+        },
+      },
+      test_dataset: {
+        class_path: 'HfPairClassificationDataset',
+        init_args: {
+          path: 'sbintuitions/JMTEB',
+          split: 'test',
+          name: 'paws_x_ja',
+        },
+      },
+    },
+  },
+}