task-comparison-by-region

app.py

@@ -28,7 +28,7 @@ from src.display.utils import (
     Precision, REGION_MAP
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_regional_comparison_df
 from src.submission.submit import handle_csv_submission
 
 text_sample_path = "src/submission_samples/model_name_text.csv"
@@ -123,6 +123,24 @@ leaderboard_dataframes_speech = {
     )
     for region in REGION_MAP.values()
 }
+
+# Pre-generate comparison dataframes
+default_comparison_regions = ["All", "Africa", "Asia (S)"]
+text_comparison_df = get_regional_comparison_df(
+    EVAL_RESULTS_PATH,
+    EVAL_REQUESTS_PATH,
+    default_comparison_regions,
+    BENCHMARK_COLS,
+    result_type="text"
+)
+
+speech_comparison_df = get_regional_comparison_df(
+    EVAL_RESULTS_PATH,
+    EVAL_REQUESTS_PATH,
+    default_comparison_regions,
+    SPEECH_BENCHMARK_COLS,
+    result_type="speech"
+)
 # Preload leaderboard blocks
 js_switch_code = """
 (displayRegion) => {
@@ -142,9 +160,9 @@ js_switch_code = """
     const target = document.getElementById("leaderboard-" + region);
     if (target) {
         target.classList.add("visible");
-        // 🧠 Trigger reflow to fix row cutoff
-        void target.offsetHeight;  // Trigger reflow
-        target.style.display = "none"; // Hide momentarily
+        // Trigger reflow to fix row cutoff
+        void target.offsetHeight;
+        target.style.display = "none";
         requestAnimationFrame(() => {
             target.style.display = "";
         });
@@ -180,6 +198,44 @@ with demo:
                 js=js_switch_code.replace("leaderboard-", "speech-leaderboard-"),
                 inputs=[speech_region_dropdown]
             )
+
+            # Regional comparison section for speech
+            with gr.Row():
+                gr.Markdown("### 🔍 Compare Regions (Average All Tasks)", elem_classes="markdown-text")
+
+            with gr.Row():
+                speech_compare_regions = gr.CheckboxGroup(
+                    choices=list(REGION_MAP.keys()),
+                    label="Select regions to compare",
+                    value=default_comparison_regions,
+                    interactive=True,
+                )
+
+            # Use Dataframe for dynamic columns
+            speech_comparison_output = gr.Dataframe(
+                value=speech_comparison_df,
+                interactive=False,
+                wrap=True,
+                elem_id="speech-comparison-table"
+            )
+
+            def update_speech_comparison(selected_regions):
+                if not selected_regions:
+                    return pd.DataFrame()
+                df = get_regional_comparison_df(
+                    EVAL_RESULTS_PATH,
+                    EVAL_REQUESTS_PATH,
+                    selected_regions,
+                    SPEECH_BENCHMARK_COLS,
+                    result_type="speech"
+                )
+                return df
+
+            speech_compare_regions.change(
+                update_speech_comparison,
+                inputs=[speech_compare_regions],
+                outputs=[speech_comparison_output]
+            )
         with gr.TabItem("🏅 mSTEB Text Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 region_dropdown = gr.Dropdown(
@@ -200,6 +256,44 @@ with demo:
             # JS hook to toggle visible leaderboard
             region_dropdown.change(None, js=js_switch_code, inputs=[region_dropdown])
 
+            # Regional comparison section for text
+            with gr.Row():
+                gr.Markdown("### 🔍 Compare Regions (Average All Tasks)", elem_classes="markdown-text")
+
+            with gr.Row():
+                text_compare_regions = gr.CheckboxGroup(
+                    choices=list(REGION_MAP.keys()),
+                    label="Select regions to compare",
+                    value=default_comparison_regions,
+                    interactive=True,
+                )
+
+            # Use Dataframe for dynamic columns
+            text_comparison_output = gr.Dataframe(
+                value=text_comparison_df,
+                interactive=False,
+                wrap=True,
+                elem_id="text-comparison-table"
+            )
+
+            def update_text_comparison(selected_regions):
+                if not selected_regions:
+                    return pd.DataFrame()
+                df = get_regional_comparison_df(
+                    EVAL_RESULTS_PATH,
+                    EVAL_REQUESTS_PATH,
+                    selected_regions,
+                    BENCHMARK_COLS,
+                    result_type="text"
+                )
+                return df
+
+            text_compare_regions.change(
+                update_text_comparison,
+                inputs=[text_compare_regions],
+                outputs=[text_comparison_output]
+            )
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

requirements.txt

@@ -1,10 +1,10 @@
 APScheduler
 black
 datasets
-gradio
+gradio==5.4.0
 gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
+gradio_leaderboard==0.0.14
+gradio_client==1.4.2
 huggingface-hub>=0.18.0
 matplotlib
 numpy

src/about.py

@@ -11,22 +11,22 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("lid", "acc", "LID")
-    task1 = Task("topic_classification", "acc", "TC")
-    task2 = Task("rc_qa", "acc", "RC-QA")
-    task3 = Task("nli", "acc", "NLI")
-    task4 = Task("machine_translation_xx_eng", "chrf", "MT (xx-en)")
-    task5 = Task("machine_translation_eng_xx", "chrf", "MT (en-xx)")
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("lid", "acc", "LID (ACC)")
+    task1 = Task("topic_classification", "acc", "TC (ACC)")
+    task2 = Task("rc_qa", "acc", "RC-QA (ACC)")
+    task3 = Task("nli", "acc", "NLI (ACC)")
+    task4 = Task("machine_translation_xx_eng", "chrf", "MT (xx-en) (CHRF++)")
+    task5 = Task("machine_translation_eng_xx", "chrf", "MT (en-xx) (CHRF++)")
 
 class SpeechTasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("lid", "acc", "LID")
-    task1 = Task("topic_classification", "acc", "TC")
-    task2 = Task("rc_qa", "acc", "RC-QA")
-    task3 = Task("asr", "cer", "ASR")
-    task4 = Task("s2tt_xx_eng", "chrf", "S2TT (xx-en)")
-    #task5 = Task("s2tt_eng_xx", "chrf", "S2TT (en-xx)")
+    task0 = Task("lid", "acc", "LID (ACC)")
+    task1 = Task("topic_classification", "acc", "TC (ACC)")
+    task2 = Task("rc_qa", "acc", "RC-QA (ACC)")
+    task3 = Task("asr", "cer", "ASR (100-CER)")
+    task4 = Task("s2tt_xx_eng", "chrf", "S2TT (xx-en) (CHRF++)")
+    #task5 = Task("s2tt_eng_xx", "chrf", "S2TT (en-xx) (CHRF++)")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/css_html_js.py

@@ -124,6 +124,26 @@ custom_css = """
 #speech-leaderboard-Europe_E.visible {
     display: block;
 }
+
+/* Target Gradio Dataframe headers with correct selectors */
+#speech-comparison-table table thead th,
+#text-comparison-table table thead th {
+    font-weight: 400 !important;
+    font-size: 14px !important;
+    padding: 6px !important;
+}
+
+/* Target with wrapper class */
+#speech-comparison-table .table-wrap table thead th,
+#text-comparison-table .table-wrap table thead th {
+    font-weight: 400 !important;
+    font-size: 14px !important;
+}
+
+/* Global table header override */
+table thead th {
+    font-weight: 400 !important;
+}
 """
 
 get_window_url_params = """

src/display/utils.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
 
 import pandas as pd
@@ -22,24 +22,36 @@ class ColumnContent:
     never_hidden: bool = False
 
 ## Leaderboard columns
+# Store column content instances separately
+auto_eval_column_content = {}
 auto_eval_column_dict = []
 # Init
-# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Model Type", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# auto_eval_column_content["model_type_symbol"] = ColumnContent("Model Type", "str", True, never_hidden=True)
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent])
+auto_eval_column_content["model"] = ColumnContent("Model", "markdown", True, never_hidden=True)
+auto_eval_column_dict.append(["model", ColumnContent])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
+auto_eval_column_content["average"] = ColumnContent("Average ⬆️", "number", True)
+auto_eval_column_dict.append(["average", ColumnContent])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_content[task.name] = ColumnContent(task.value.col_name, "number", True)
+    auto_eval_column_dict.append([task.name, ColumnContent])
 
 ### Speech leaderboard columns
+# Store column content instances separately
+auto_eval_column_content_speech = {}
 auto_eval_column_dict_speech = []
 # Init
-# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Model Type", "str", True, never_hidden=True)])
-auto_eval_column_dict_speech.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# auto_eval_column_content_speech["model_type_symbol"] = ColumnContent("Model Type", "str", True, never_hidden=True)
+# auto_eval_column_dict_speech.append(["model_type_symbol", ColumnContent])
+auto_eval_column_content_speech["model"] = ColumnContent("Model", "markdown", True, never_hidden=True)
+auto_eval_column_dict_speech.append(["model", ColumnContent])
 #Scores
-auto_eval_column_dict_speech.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
+auto_eval_column_content_speech["average"] = ColumnContent("Average ⬆️", "number", True)
+auto_eval_column_dict_speech.append(["average", ColumnContent])
 for task in SpeechTasks:
-    auto_eval_column_dict_speech.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_content_speech[task.name] = ColumnContent(task.value.col_name, "number", True)
+    auto_eval_column_dict_speech.append([task.name, ColumnContent])
 
 
 # Model information
@@ -55,10 +67,16 @@ for task in SpeechTasks:
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+# Set class attributes with the ColumnContent instances
+for col_name, col_content in auto_eval_column_content.items():
+    setattr(AutoEvalColumn, col_name, col_content)
+
 AutoEvalColumnSpeech = make_dataclass("AutoEvalColumnSpeech", auto_eval_column_dict_speech, frozen=True)
+# Set class attributes with the ColumnContent instances
+for col_name, col_content in auto_eval_column_content_speech.items():
+    setattr(AutoEvalColumnSpeech, col_name, col_content)
 
 ## For the queue columns in the submission tab
-@dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)

src/leaderboard/read_evals.py

@@ -129,20 +129,36 @@ class EvalResult:
 
     def to_dict(self, region=None, result_type='text'):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        # print(self.results)
         task_enum = Tasks if result_type == "text" else SpeechTasks
 
         results = self.results if region is None else self.regions.get(region, {})
-        acc_values = [
-            results[task.value.benchmark]
-            for task in task_enum
-            if task.value.metric == "acc" and task.value.benchmark in results
-        ]
-        # print(acc_values)
 
-        average = sum(acc_values) / len(acc_values) if acc_values else None
+        # Calculate average for ALL tasks (main)
+        all_scores = []
+        for task in task_enum:
+            if task.value.benchmark in results:
+                result_dict = results[task.value.benchmark]
+                # Handle nested dict structure (e.g., {'acc': 0.812})
+                if isinstance(result_dict, dict) and task.value.metric in result_dict:
+                    score = result_dict[task.value.metric]
+                    # Convert to percentage if needed
+                    if score <= 1:
+                        score = score * 100
+                    # Convert CER to 100-CER for average calculation
+                    if task.value.metric == "cer":
+                        score = 100 - score
+                    all_scores.append(score)
+                elif isinstance(result_dict, (int, float)):
+                    score = result_dict
+                    if score <= 1:
+                        score = score * 100
+                    # Convert CER to 100-CER for average calculation
+                    if task.value.metric == "cer":
+                        score = 100 - score
+                    all_scores.append(score)
+
+        average = sum(all_scores) / len(all_scores) if all_scores else None
 
-        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             # AutoEvalColumn.precision.name: self.precision.value.name,
@@ -161,9 +177,31 @@ class EvalResult:
 
         for task in task_enum:
             if task.value.benchmark in results:
-                data_dict[task.value.col_name] = results[task.value.benchmark]
+                result_dict = results[task.value.benchmark]
+                # Handle nested dict structure
+                if isinstance(result_dict, dict) and task.value.metric in result_dict:
+                    score = result_dict[task.value.metric]
+                    # Convert to percentage if needed
+                    if task.value.metric in ["acc", "chrf"] and score <= 1:
+                        score = score * 100
+                    elif task.value.metric == "cer":
+                        if score <= 1:
+                            score = score * 100
+                        # Convert CER to 100-CER
+                        score = 100 - score
+                    data_dict[task.value.col_name] = score
+                elif isinstance(result_dict, (int, float)):
+                    score = result_dict
+                    # Convert CER to 100-CER
+                    if task.value.metric == "cer":
+                        if score <= 1:
+                            score = score * 100
+                        score = 100 - score
+                    data_dict[task.value.col_name] = score
+                else:
+                    data_dict[task.value.col_name] = None
             else:
-                data_dict[task.value.col_name] = None  # or np.nan if preferred
+                data_dict[task.value.col_name] = None
 
         return data_dict
 
@@ -217,8 +255,6 @@ def get_raw_eval_results(results_path: str, requests_path: str, result_type: str
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath,result_type)
-        # print('testing this one')
-        # print(eval_result)
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
@@ -231,11 +267,9 @@ def get_raw_eval_results(results_path: str, requests_path: str, result_type: str
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()
             results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError:
             continue
-    # print('results')
-    # print(results)
 
     return results

src/populate.py

@@ -4,17 +4,14 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model, has_at_least_one_benchmark
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, REGION_MAP
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, region=None, result_type="text") -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path, result_type=result_type)
-    # this here if region is none gets main results. I have to pass region value here to get region based results
-    # and they should come.
     all_data_json = [v.to_dict(region, result_type) for v in raw_data]
-    # print('all_data_json', all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
@@ -57,3 +54,91 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
     return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+def get_regional_comparison_df(results_path: str, requests_path: str, selected_regions: list, benchmark_cols: list, result_type="text") -> pd.DataFrame:
+    """Creates a comparison dataframe"""
+    if not selected_regions or len(selected_regions) < 1:
+        return pd.DataFrame()
+
+    raw_data = get_raw_eval_results(results_path, requests_path, result_type=result_type)
+    from src.about import Tasks, SpeechTasks
+
+    task_enum = Tasks if result_type == "text" else SpeechTasks
+
+    region_short_names = {
+        "All": "All",
+        "Africa": "Africa",
+        "Americas/Oceania": "Americas/Oceania",
+        "Asia (S)": "Asia (S)",
+        "Asia (SE)": "Asia (SE)",
+        "Asia (W, C)": "Asia (W,C)",
+        "Asia (E)": "Asia (E)",
+        "Europe (W, N, S)": "Europe (W,N,S)",
+        "Europe (E)": "Europe (E)",
+    }
+
+    # Build comparison data
+    comparison_data = []
+    for eval_result in raw_data:
+        row = {
+            "Model": eval_result.full_model,
+        }
+
+        # Add average scores for each selected region
+        for region in selected_regions:
+            region_key = REGION_MAP.get(region, region)
+
+            # Get region results
+            if region_key == "All":
+                results = eval_result.results
+            else:
+                results = eval_result.regions.get(region_key, {}) if eval_result.regions else {}
+
+            # Calculate average across ALL tasks for comparison table
+            all_scores = []
+            for task in task_enum:
+                if task.value.benchmark in results:
+                    result_dict = results[task.value.benchmark]
+                    # Handle nested dict structure
+                    if isinstance(result_dict, dict) and task.value.metric in result_dict:
+                        score = result_dict[task.value.metric]
+                    elif isinstance(result_dict, (int, float)):
+                        score = result_dict
+                    else:
+                        continue
+
+                    if score is not None:
+                        # Convert to percentage if needed
+                        if task.value.metric in ["acc", "chrf"] and score <= 1:
+                            score = score * 100
+                        elif task.value.metric == "cer":
+                            if score <= 1:
+                                score = score * 100
+                            score = 100 - score  # Convert CER to accuracy-like metric
+                        all_scores.append(score)
+
+            # Calculate average of all tasks
+            avg_all_tasks = sum(all_scores) / len(all_scores) if all_scores else None
+
+            # Use shortened region name
+            short_name = region_short_names.get(region, region)
+            row[short_name] = avg_all_tasks
+
+        comparison_data.append(row)
+
+    if not comparison_data:
+        return pd.DataFrame()
+
+    df = pd.DataFrame(comparison_data)
+
+    # Sort by the first region's average score
+    first_region_short = region_short_names.get(selected_regions[0], selected_regions[0])
+    if first_region_short in df.columns:
+        df = df.sort_values(by=[first_region_short], ascending=False)
+
+    # Round numerical columns
+    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
+    df[numeric_cols] = df[numeric_cols].round(decimals=2)
+
+    return df

src/submission/submit.py

@@ -41,13 +41,11 @@ def handle_csv_submission(
         path_or_fileobj=csv_save_path,
         path_in_repo=remote_path,
         repo_id=QUEUE_REPO,
-        repo_type="dataset",  # or "model" if you made the repo that way
+        repo_type="dataset",
         commit_message=f"Add {result_type} request for {model_name} at {current_time}",
     )
 
-    # Remove the local file
     os.remove(csv_save_path)
-    # this converts dataframe to json and uploads it to results
 
 
     try:
@@ -83,7 +81,6 @@ def convert_csv_to_json_and_upload(df: pd.DataFrame, model_name: str, result_typ
             raise ValueError(f"Region '{region}' not found in REGION_MAP keys.")
 
     # --- Build JSON ---
-    # I go over the regions in the CSV and create a JSON object.
     model_json = {
         "config": {"model_name": model_name},
         "results": {},
@@ -100,7 +97,6 @@ def convert_csv_to_json_and_upload(df: pd.DataFrame, model_name: str, result_typ
                     continue
                 task = find_task_by_col_name(col, task_enum)
                 if val is not None and not pd.isna(val) and isinstance(val, (int, float)):
-                    print(f" value {val}")
                     at_least_one_number = True
                     model_json["results"][task.value.benchmark] = {task.value.metric: val/100}
         else:
@@ -112,8 +108,6 @@ def convert_csv_to_json_and_upload(df: pd.DataFrame, model_name: str, result_typ
                 if val is not None and not pd.isna(val) and isinstance(val, (int, float)):
                     model_json["regions"][REGION_MAP[region_display]][task.value.benchmark] = {task.value.metric: val/100}
 
-    # Check if at least one number is present in the results
-    print(at_least_one_number)
     if at_least_one_number is False:
         raise ValueError("No valid numeric results found in the CSV. Please check your input.")