Pairing improvements: filter already-evaluated pairs from /data, round-robin schedule across test_ids, alternate A/B order per pair; ensure submit maps scores to correct model columns and auto-advance

app.py

@@ -1,5 +1,6 @@
 import csv
 import itertools
+import random
 import json
 import os
 import uuid
@@ -15,6 +16,7 @@ except Exception:  # optional dependency at runtime
 
 
 BASE_DIR = os.path.dirname(__file__)
+PERSIST_DIR = os.environ.get("PERSIST_DIR", "/data")
 # Persistent local storage inside HF Spaces
 PERSIST_DIR = os.environ.get("PERSIST_DIR", "/data")
 TASK_CONFIG = {
@@ -35,6 +37,11 @@ def _csv_path_for_task(task_name: str, filename: str) -> str:
     return os.path.join(BASE_DIR, folder, filename)
 
 
+def _persist_csv_path_for_task(task_name: str) -> str:
+    folder = TASK_CONFIG[task_name]["folder"]
+    return os.path.join(PERSIST_DIR, folder, "evaluation_results.csv")
+
+
 def _resolve_image_path(path: str) -> str:
     return path if os.path.isabs(path) else os.path.join(BASE_DIR, path)
 
@@ -87,12 +94,72 @@ def _build_image_pairs(rows: List[Dict[str, str]], task_name: str) -> List[Dict[
     return pairs
 
 
+def _read_existing_eval_keys(task_name: str) -> set:
+    """Read already-evaluated pair keys from persistent CSV, return a set of keys.
+    Key is (test_id, frozenset({model1_name, model2_name}), org_img) to ignore A/B order.
+    """
+    keys = set()
+    csv_path = _persist_csv_path_for_task(task_name)
+    if not os.path.exists(csv_path):
+        return keys
+    try:
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for r in reader:
+                tid = str(r.get("test_id", "")).strip()
+                m1 = str(r.get("model1_name", "")).strip()
+                m2 = str(r.get("model2_name", "")).strip()
+                org = str(r.get("org_img", "")).strip()
+                if tid and m1 and m2 and org:
+                    keys.add((tid, frozenset({m1, m2}), org))
+    except Exception:
+        pass
+    return keys
+
+
+def _schedule_round_robin_by_test_id(pairs: List[Dict[str, str]], seed: int | None = None) -> List[Dict[str, str]]:
+    """Interleave pairs across test_ids for balanced coverage; shuffle within each group.
+    """
+    groups: Dict[str, List[Dict[str, str]]] = {}
+    for p in pairs:
+        groups.setdefault(p["test_id"], []).append(p)
+    rnd = random.Random(seed)
+    for lst in groups.values():
+        rnd.shuffle(lst)
+    # round-robin drain
+    ordered: List[Dict[str, str]] = []
+    while True:
+        progressed = False
+        for tid in sorted(groups.keys(), key=lambda x: (int(x) if x.isdigit() else x)):
+            if groups[tid]:
+                ordered.append(groups[tid].pop())
+                progressed = True
+        if not progressed:
+            break
+    return ordered
+
+
 def load_task(task_name: str):
     if not task_name:
         raise gr.Error("请先选择任务。")
 
     rows = _load_task_rows(task_name)
     pairs = _build_image_pairs(rows, task_name)
+    # Filter out already evaluated pairs from persistent CSV
+    done_keys = _read_existing_eval_keys(task_name)
+    def key_of(p: Dict[str, str]):
+        return (p["test_id"], frozenset({p["model1_name"], p["model2_name"]}), p["org_img"])
+    pairs = [p for p in pairs if key_of(p) not in done_keys]
+
+    # Balanced schedule across test_ids with a stable randomization
+    seed_env = os.environ.get("SCHEDULE_SEED")
+    seed = int(seed_env) if seed_env and seed_env.isdigit() else None
+    pairs = _schedule_round_robin_by_test_id(pairs, seed=seed)
+
+    # Assign A/B order to counteract position bias: alternate after scheduling
+    for idx, p in enumerate(pairs):
+        p["swap"] = bool(idx % 2)  # True -> A=B's image; False -> A=A's image
+
     if not pairs:
         raise gr.Error("没有找到可评测的图片对，请检查数据文件。")
 
@@ -204,13 +271,16 @@ def on_task_change(task_name: str, _state_pairs: List[Dict[str, str]]):
     header = _format_pair_header(pair)
     # Defaults for A and B (8 sliders total)
     default_scores = [3, 3, 3, 3, 3, 3, 3, 3]
+    # Pick display order according to swap flag
+    a_path = pair["model2_path"] if pair.get("swap") else pair["model1_path"]
+    b_path = pair["model1_path"] if pair.get("swap") else pair["model2_path"]
     return (
         pairs,
         gr.update(value=0, minimum=0, maximum=len(pairs) - 1, visible=(len(pairs) > 1)),
         gr.update(value=header),
         _resolve_image_path(pair["org_img"]),
-        _resolve_image_path(pair["model1_path"]),
-        _resolve_image_path(pair["model2_path"]),
+        _resolve_image_path(a_path),
+        _resolve_image_path(b_path),
         *default_scores,
         gr.update(value=f"共 {len(pairs)} 个待评测的图片对。"),
     )
@@ -223,12 +293,14 @@ def on_pair_navigate(index: int, pairs: List[Dict[str, str]]):
     index = max(0, min(index, len(pairs) - 1))
     pair = pairs[index]
     header = _format_pair_header(pair)
+    a_path = pair["model2_path"] if pair.get("swap") else pair["model1_path"]
+    b_path = pair["model1_path"] if pair.get("swap") else pair["model2_path"]
     return (
         gr.update(value=index),
         gr.update(value=header),
         _resolve_image_path(pair["org_img"]),
-        _resolve_image_path(pair["model1_path"]),
-        _resolve_image_path(pair["model2_path"]),
+        _resolve_image_path(a_path),
+        _resolve_image_path(b_path),
         3, 3, 3, 3,  # A
         3, 3, 3, 3,  # B
     )
@@ -266,6 +338,19 @@ def on_submit(
         "model2_semantic_functional_alignment_score": int(b_semantic_score),
         "model2_overall_photorealism_score": int(b_overall_score),
     }
+    # Map A/B scores to the correct model columns depending on swap
+    if pair.get("swap"):
+        # UI A == model2, UI B == model1
+        score_map = {
+            "model1_physical_interaction_fidelity_score": int(b_physical_score),
+            "model1_optical_effect_accuracy_score": int(b_optical_score),
+            "model1_semantic_functional_alignment_score": int(b_semantic_score),
+            "model1_overall_photorealism_score": int(b_overall_score),
+            "model2_physical_interaction_fidelity_score": int(a_physical_score),
+            "model2_optical_effect_accuracy_score": int(a_optical_score),
+            "model2_semantic_functional_alignment_score": int(a_semantic_score),
+            "model2_overall_photorealism_score": int(a_overall_score),
+        }
     row = _build_eval_row(pair, score_map)
     ok_local = _append_local_persist_csv(task_name, row)
     ok_hub, hub_msg = _upload_eval_record_to_dataset(task_name, row)
@@ -278,12 +363,14 @@ def on_submit(
     if next_index != index:
         pair = pairs[next_index]
         header = _format_pair_header(pair)
+        a_path = pair["model2_path"] if pair.get("swap") else pair["model1_path"]
+        b_path = pair["model1_path"] if pair.get("swap") else pair["model2_path"]
         return (
             gr.update(value=next_index),
             gr.update(value=header),
             _resolve_image_path(pair["org_img"]),
-            _resolve_image_path(pair["model1_path"]),
-            _resolve_image_path(pair["model2_path"]),
+            _resolve_image_path(a_path),
+            _resolve_image_path(b_path),
             3, 3, 3, 3,
             3, 3, 3, 3,
             gr.update(value=info + f" 自动跳转到下一组（{next_index + 1}/{len(pairs)}）。"),