Add speech tab

app.py

@@ -15,10 +15,13 @@ from src.about import (
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
+    SPEECH_BENCHMARK_COLS,
     COLS,
+    COLS_SPEECH,
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
+    AutoEvalColumnSpeech,
     ModelType,
     fields,
     WeightType,
@@ -58,20 +61,22 @@ except Exception:
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-def init_leaderboard(dataframe):
+def init_leaderboard(dataframe,result_type='text'):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
+    column_class = AutoEvalColumn if result_type == "text" else AutoEvalColumnSpeech
+
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=[c.type for c in fields(column_class)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=[c.name for c in fields(column_class) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(column_class) if c.never_hidden],
             label="Select Columns to Display:",
         ),
         # search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        search_columns=[AutoEvalColumn.model.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        search_columns=[column_class.model.name],
+        hide_columns=[c.name for c in fields(column_class) if c.hidden],
         filter_columns=[
             # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
             # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
@@ -111,6 +116,19 @@ leaderboard_dataframes = {
         COLS,
         BENCHMARK_COLS,
         region if region != "All" else None,
+        result_type="text"
+    )
+    for region in REGIONS
+}
+
+leaderboard_dataframes_speech = {
+    region: get_leaderboard_df(
+        EVAL_RESULTS_PATH,
+        EVAL_REQUESTS_PATH,
+        COLS_SPEECH,
+        SPEECH_BENCHMARK_COLS,
+        region if region != "All" else None,
+        result_type="speech"
     )
     for region in REGIONS
 }
@@ -165,10 +183,32 @@ with demo:
                         elem_id=f"leaderboard-{region_key}",
                         elem_classes=["visible"] if region_key == "All" else []
                 ):
-                    init_leaderboard(leaderboard_dataframes[region_key])
+                    init_leaderboard(leaderboard_dataframes[region_key], result_type="text")
 
             # JS hook to toggle visible leaderboard
             region_dropdown.change(None, js=js_switch_code, inputs=[region_dropdown])
+
+        with gr.TabItem("🗣️ mSTEB Speech Benchmark", elem_id="speech-benchmark-tab-table", id=1):
+            with gr.Row():
+                speech_region_dropdown = gr.Dropdown(
+                    choices=list(REGION_MAP.keys()),
+                    label="Select Region",
+                    value="All",
+                    interactive=True,
+                )
+
+            for display_name, region_key in REGION_MAP.items():
+                with gr.Column(
+                        elem_id=f"speech-leaderboard-{region_key}",
+                        elem_classes=["visible"] if region_key == "All" else []
+                ):
+                    init_leaderboard(leaderboard_dataframes_speech[region_key],result_type='speech')
+
+            speech_region_dropdown.change(
+                None,
+                js=js_switch_code.replace("leaderboard-", "speech-leaderboard-"),
+                inputs=[speech_region_dropdown]
+            )
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

src/about.py

@@ -19,6 +19,14 @@ class Tasks(Enum):
     task4 = Task("machine_translation_xx_eng", "chrf", "MT (xx-en)")
     task5 = Task("machine_translation_eng_xx", "chrf", "MT (en-xx)")
 
+class SpeechTasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("lid", "acc", "LID")
+    task1 = Task("topic_classification", "acc", "TC")
+    task2 = Task("rc_qa", "acc", "RC-QA")
+    task3 = Task("asr", "cer", "ASR")
+    task4 = Task("s2tt", "chrf", "S2TT")
+
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/display/css_html_js.py

@@ -109,6 +109,21 @@ custom_css = """
     display: block;
 }
 
+
+[id^="speech-leaderboard-"] {
+    display: none;
+}
+#speech-leaderboard-All.visible,
+#speech-leaderboard-Africa.visible,
+#speech-leaderboard-Americas_Oceania.visible,
+#speech-leaderboard-Asia_S.visible,
+#speech-leaderboard-Asia_SE.visible,
+#speech-leaderboard-Asia_W_C.visible,
+#speech-leaderboard-Asia_E.visible,
+#speech-leaderboard-Europe_W_N_S.visible,
+#speech-leaderboard-Europe_E.visible {
+    display: block;
+}
 """
 
 get_window_url_params = """

src/display/utils.py

@@ -4,6 +4,7 @@ from enum import Enum
 import pandas as pd
 
 from src.about import Tasks
+from src.about import SpeechTasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -29,6 +30,18 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+### Speech leaderboard columns
+auto_eval_column_dict_speech = []
+# Init
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Model Type", "str", True, never_hidden=True)])
+auto_eval_column_dict_speech.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict_speech.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
+for task in SpeechTasks:
+    auto_eval_column_dict_speech.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+
 # Model information
 # auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 # auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -42,6 +55,7 @@ for task in Tasks:
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+AutoEvalColumnSpeech = make_dataclass("AutoEvalColumnSpeech", auto_eval_column_dict_speech, frozen=True)
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -102,9 +116,10 @@ class Precision(Enum):
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_SPEECH = [c.name for c in fields(AutoEvalColumnSpeech) if not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
+SPEECH_BENCHMARK_COLS = [t.value.col_name for t in SpeechTasks]

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, SpeechTasks
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -34,7 +34,7 @@ class EvalResult:
     regions: dict = None
 
     @classmethod
-    def init_from_json_file(self, json_filepath):
+    def init_from_json_file(self, json_filepath, result_type='speech'):
         """Inits the result from the specific model result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
@@ -70,7 +70,10 @@ class EvalResult:
 
         # Extract results available in this file (some results are split in several files)
         results = {}
-        for task in Tasks:
+
+        task_enum = Tasks if result_type == "text" else SpeechTasks
+
+        for task in task_enum:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
@@ -84,7 +87,7 @@ class EvalResult:
         regions_processed_results = {}
         for region, region_results in regions.items():
             processed = {}
-            for task in Tasks:
+            for task in task_enum:
                 task = task.value
 
                 # We average all scores of a given metric (not all metrics are present in all files)
@@ -124,13 +127,15 @@ class EvalResult:
         except Exception:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
-    def to_dict(self, region=None):
+    def to_dict(self, region=None, result_type='text'):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         # print(self.results)
+        task_enum = Tasks if result_type == "text" else SpeechTasks
+
         results = self.results if region is None else self.regions.get(region, {})
         acc_values = [
             results[task.value.benchmark]
-            for task in Tasks
+            for task in task_enum
             if task.value.metric == "acc" and task.value.benchmark in results
         ]
         # print(acc_values)
@@ -154,7 +159,7 @@ class EvalResult:
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
+        for task in task_enum:
             if task.value.benchmark in results:
                 data_dict[task.value.col_name] = results[task.value.benchmark]
             else:
@@ -185,12 +190,17 @@ def get_request_file_for_model(requests_path, model_name, precision):
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str, requests_path: str, result_type: str = "text") -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
+    # result type
     model_result_filepaths = []
 
     for root, _, files in os.walk(results_path):
         # We should only have json files in model results
+        if result_type == "text" and "msteb_text_results" not in root:
+            continue
+        if result_type == "speech" and "msteb_speech_results" not in root:
+            continue
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
 
@@ -206,7 +216,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result = EvalResult.init_from_json_file(model_result_filepath,result_type)
         # print('testing this one')
         # print(eval_result)
         eval_result.update_with_request_file(requests_path)

src/populate.py

@@ -8,12 +8,12 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, region=None) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, region=None, result_type="text") -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(results_path, requests_path, result_type=result_type)
     # this here if region is none gets main results. I have to pass region value here to get region based results
     # and they should come.
-    all_data_json = [v.to_dict(region) for v in raw_data]
+    all_data_json = [v.to_dict(region, result_type) for v in raw_data]
     # print('all_data_json', all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)