Spaces:
Runtime error
Runtime error
aether-raider commited on
Commit ·
653a3eb
1
Parent(s): 41cd6b6
tested and completed mos form
Browse files- backend/__pycache__/session_manager.cpython-311.pyc +0 -0
- backend/session_manager.py +65 -28
- frontend/app.py +72 -219
- frontend/pages/__pycache__/ab_gender.cpython-311.pyc +0 -0
- frontend/pages/__pycache__/ab_model.cpython-311.pyc +0 -0
- frontend/pages/__pycache__/conclusion.cpython-311.pyc +0 -0
- frontend/pages/__pycache__/mos.cpython-311.pyc +0 -0
- frontend/pages/ab_gender.py +10 -0
- frontend/pages/ab_model.py +10 -0
- frontend/pages/conclusion.py +10 -0
- frontend/pages/mos.py +10 -0
- frontend/utils/__pycache__/data_persistence.cpython-311.pyc +0 -0
- frontend/utils/__pycache__/js_collectors.cpython-311.pyc +0 -0
- frontend/utils/__pycache__/validation.cpython-311.pyc +0 -0
- frontend/utils/js_collectors.py +148 -0
- frontend/utils/validation.py +113 -0
backend/__pycache__/session_manager.cpython-311.pyc
CHANGED
|
Binary files a/backend/__pycache__/session_manager.cpython-311.pyc and b/backend/__pycache__/session_manager.cpython-311.pyc differ
|
|
|
backend/session_manager.py
CHANGED
|
@@ -43,17 +43,17 @@ class SessionManager:
|
|
| 43 |
exercise_groups[clip.exercise_id] = {"male": [], "female": []}
|
| 44 |
exercise_groups[clip.exercise_id][clip.speaker].append(clip)
|
| 45 |
|
| 46 |
-
|
|
|
|
| 47 |
for _, speakers in exercise_groups.items():
|
| 48 |
-
if speakers["male"]
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
|
| 53 |
-
# Select 3 random
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
mos_clips.extend([male_clip, female_clip])
|
| 57 |
|
| 58 |
# Group by content (exercise + transcript) for comparisons
|
| 59 |
content_groups: Dict[Any, List[Clip]] = {}
|
|
@@ -213,14 +213,18 @@ class SessionManager:
|
|
| 213 |
return
|
| 214 |
|
| 215 |
try:
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
mos_response = {
|
| 225 |
"session_id": session["session_id"],
|
| 226 |
"clip_id": clip_id,
|
|
@@ -245,7 +249,14 @@ class SessionManager:
|
|
| 245 |
}
|
| 246 |
|
| 247 |
self.save_response("mos", mos_response)
|
| 248 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
|
| 250 |
except Exception as e:
|
| 251 |
print(f"[WARN] Error processing MOS data: {e}")
|
|
@@ -272,16 +283,42 @@ class SessionManager:
|
|
| 272 |
return
|
| 273 |
|
| 274 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
for comp_id, comparison in comparisons_data.items():
|
| 276 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
continue
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
ab_response = {
|
| 280 |
-
"session_id":
|
| 281 |
-
"clip_a_id":
|
| 282 |
-
"clip_b_id":
|
| 283 |
"comparison_type": comparison.get("comparison_type"),
|
| 284 |
-
"choice": comparison.get("choice"),
|
| 285 |
"comment": comparison.get("comment", ""),
|
| 286 |
# Support both model_vs_model (gender_mismatch_a/b)
|
| 287 |
# and gender_vs_gender (gender_mismatch_male/female)
|
|
@@ -293,10 +330,10 @@ class SessionManager:
|
|
| 293 |
}
|
| 294 |
|
| 295 |
self.save_response("ab", ab_response)
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
)
|
| 300 |
|
| 301 |
except Exception as e:
|
| 302 |
print(f"[WARN] Error processing A/B data: {e}")
|
|
|
|
| 43 |
exercise_groups[clip.exercise_id] = {"male": [], "female": []}
|
| 44 |
exercise_groups[clip.exercise_id][clip.speaker].append(clip)
|
| 45 |
|
| 46 |
+
# Collect all clips from this model
|
| 47 |
+
all_model_clips = []
|
| 48 |
for _, speakers in exercise_groups.items():
|
| 49 |
+
if speakers["male"]:
|
| 50 |
+
all_model_clips.extend(speakers["male"])
|
| 51 |
+
if speakers["female"]:
|
| 52 |
+
all_model_clips.extend(speakers["female"])
|
| 53 |
|
| 54 |
+
# Select 3 random clips (regardless of gender pairing) for this model
|
| 55 |
+
selected_clips = rng.sample(all_model_clips, min(3, len(all_model_clips)))
|
| 56 |
+
mos_clips.extend(selected_clips)
|
|
|
|
| 57 |
|
| 58 |
# Group by content (exercise + transcript) for comparisons
|
| 59 |
content_groups: Dict[Any, List[Clip]] = {}
|
|
|
|
| 213 |
return
|
| 214 |
|
| 215 |
try:
|
| 216 |
+
# Get all clips that were presented to the user
|
| 217 |
+
presented_clips = session.get("mos_clips", [])
|
| 218 |
+
presented_clip_ids = {clip.id for clip in presented_clips}
|
| 219 |
+
|
| 220 |
+
print(f"[DEBUG] Presented {len(presented_clip_ids)} MOS clips to user")
|
| 221 |
+
print(f"[DEBUG] Received ratings for {len(ratings_data)} clips")
|
| 222 |
+
|
| 223 |
+
# Process all presented clips, whether rated or not
|
| 224 |
+
for clip in presented_clips:
|
| 225 |
+
clip_id = clip.id
|
| 226 |
+
ratings = ratings_data.get(clip_id, {})
|
| 227 |
+
|
| 228 |
mos_response = {
|
| 229 |
"session_id": session["session_id"],
|
| 230 |
"clip_id": clip_id,
|
|
|
|
| 249 |
}
|
| 250 |
|
| 251 |
self.save_response("mos", mos_response)
|
| 252 |
+
|
| 253 |
+
# Log whether this clip was rated or not
|
| 254 |
+
has_ratings = any(
|
| 255 |
+
ratings.get(dim)
|
| 256 |
+
for dim in ["clarity", "pronunciation", "prosody", "naturalness", "overall"]
|
| 257 |
+
)
|
| 258 |
+
status = "rated" if has_ratings else "not rated"
|
| 259 |
+
print(f"[INFO] Processed MOS clip {clip_id} ({status})")
|
| 260 |
|
| 261 |
except Exception as e:
|
| 262 |
print(f"[WARN] Error processing MOS data: {e}")
|
|
|
|
| 283 |
return
|
| 284 |
|
| 285 |
try:
|
| 286 |
+
# Get all AB pairs that were presented to the user from the submitted data
|
| 287 |
+
# The comparison data will tell us which type based on the IDs
|
| 288 |
+
print(f"[DEBUG] Received ratings for {len(comparisons_data)} comparisons")
|
| 289 |
+
|
| 290 |
+
# Build a set of already saved comparison IDs to avoid duplicates
|
| 291 |
+
session_id = session["session_id"]
|
| 292 |
+
existing_ab_responses = [
|
| 293 |
+
r for r in self.responses.get("ab", [])
|
| 294 |
+
if r.get("session_id") == session_id
|
| 295 |
+
]
|
| 296 |
+
existing_pairs = {
|
| 297 |
+
(r["clip_a_id"], r["clip_b_id"])
|
| 298 |
+
for r in existing_ab_responses
|
| 299 |
+
}
|
| 300 |
+
print(f"[DEBUG] Already have {len(existing_pairs)} AB comparisons saved for this session")
|
| 301 |
+
|
| 302 |
+
# Process each comparison from the submitted data
|
| 303 |
for comp_id, comparison in comparisons_data.items():
|
| 304 |
+
clip_a_id = comparison.get("clip_a_id")
|
| 305 |
+
clip_b_id = comparison.get("clip_b_id")
|
| 306 |
+
|
| 307 |
+
if not clip_a_id or not clip_b_id:
|
| 308 |
+
print(f"[WARN] Skipping comparison {comp_id} - missing clip IDs")
|
| 309 |
continue
|
| 310 |
+
|
| 311 |
+
# Skip if we've already saved this pair
|
| 312 |
+
if (clip_a_id, clip_b_id) in existing_pairs:
|
| 313 |
+
print(f"[DEBUG] Skipping duplicate comparison: {clip_a_id} vs {clip_b_id}")
|
| 314 |
+
continue
|
| 315 |
+
|
| 316 |
ab_response = {
|
| 317 |
+
"session_id": session_id,
|
| 318 |
+
"clip_a_id": clip_a_id,
|
| 319 |
+
"clip_b_id": clip_b_id,
|
| 320 |
"comparison_type": comparison.get("comparison_type"),
|
| 321 |
+
"choice": comparison.get("choice"), # Can be None if not rated
|
| 322 |
"comment": comparison.get("comment", ""),
|
| 323 |
# Support both model_vs_model (gender_mismatch_a/b)
|
| 324 |
# and gender_vs_gender (gender_mismatch_male/female)
|
|
|
|
| 330 |
}
|
| 331 |
|
| 332 |
self.save_response("ab", ab_response)
|
| 333 |
+
existing_pairs.add((clip_a_id, clip_b_id)) # Mark as saved
|
| 334 |
+
|
| 335 |
+
status = "rated" if comparison.get("choice") else "not rated"
|
| 336 |
+
print(f"[INFO] Processed A/B comparison {clip_a_id} vs {clip_b_id} ({status})")
|
| 337 |
|
| 338 |
except Exception as e:
|
| 339 |
print(f"[WARN] Error processing A/B data: {e}")
|
frontend/app.py
CHANGED
|
@@ -5,7 +5,6 @@ Main entrypoint wiring backend + frontend pages.
|
|
| 5 |
|
| 6 |
import time
|
| 7 |
import gradio as gr
|
| 8 |
-
|
| 9 |
import os
|
| 10 |
import sys
|
| 11 |
|
|
@@ -17,9 +16,8 @@ sys.path.insert(0, mos_dir)
|
|
| 17 |
from backend.data_manager import DataManager
|
| 18 |
from backend.session_manager import SessionManager
|
| 19 |
from backend.hf_logging import push_session_to_hub
|
| 20 |
-
from backend.models import get_display_model_name, audio_to_base64_url
|
| 21 |
|
| 22 |
-
# Frontend
|
| 23 |
from frontend.css import CSS
|
| 24 |
from frontend.pages.intro import build_intro_page
|
| 25 |
from frontend.pages.samples import build_sample_page
|
|
@@ -28,12 +26,27 @@ from frontend.pages.ab_model import build_ab_model_page, render_ab_model_html
|
|
| 28 |
from frontend.pages.ab_gender import build_ab_gender_page, render_ab_gender_html
|
| 29 |
from frontend.pages.conclusion import build_conclusion_page
|
| 30 |
from frontend.pages.thank_you import build_thank_you_page
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def create_app(data_manager, session_manager):
|
| 34 |
with gr.Blocks(title="ATC TTS Evaluation", css=CSS) as app:
|
| 35 |
# Global state
|
| 36 |
session_state = gr.State()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
# Hidden textboxes used by the JS collectors on MOS / AB pages
|
| 39 |
mos_data_textbox = gr.Textbox(
|
|
@@ -125,11 +138,18 @@ def create_app(data_manager, session_manager):
|
|
| 125 |
)
|
| 126 |
|
| 127 |
def continue_from_mos_to_ab_model(session, mos_data_json):
|
| 128 |
-
"""
|
| 129 |
-
Process MOS data (collected via JavaScript) and move to Model-vs-Model page.
|
| 130 |
-
"""
|
| 131 |
print(f"[DEBUG] continue_from_mos_to_ab_model called")
|
| 132 |
print(f"[DEBUG] MOS data JSON received: '{mos_data_json}'")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
session_manager.process_mos_data(session, mos_data_json)
|
| 134 |
ab_model_html = render_ab_model_html(session)
|
| 135 |
return (
|
|
@@ -139,11 +159,18 @@ def create_app(data_manager, session_manager):
|
|
| 139 |
)
|
| 140 |
|
| 141 |
def continue_from_ab_model_to_ab_gender(session, ab_data_json):
|
| 142 |
-
"""
|
| 143 |
-
Process A/B model comparison data (collected via JavaScript) and move to Gender A/B page.
|
| 144 |
-
"""
|
| 145 |
print(f"[DEBUG] continue_from_ab_model_to_ab_gender called")
|
| 146 |
print(f"[DEBUG] AB data JSON received: '{ab_data_json}'")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
session_manager.process_ab_data(session, ab_data_json)
|
| 148 |
ab_gender_html = render_ab_gender_html(session)
|
| 149 |
return (
|
|
@@ -153,9 +180,18 @@ def create_app(data_manager, session_manager):
|
|
| 153 |
)
|
| 154 |
|
| 155 |
def complete_evaluation_with_ab_gender_data(session, ab_data_json):
|
| 156 |
-
"""
|
| 157 |
-
|
| 158 |
-
""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
session_manager.process_ab_data(session, ab_data_json)
|
| 160 |
if session:
|
| 161 |
session["completed"] = True
|
|
@@ -171,10 +207,7 @@ def create_app(data_manager, session_manager):
|
|
| 171 |
mos_data_json,
|
| 172 |
ab_data_json,
|
| 173 |
):
|
| 174 |
-
"""
|
| 175 |
-
Final submit: process any remaining MOS/AB data (collected via JavaScript),
|
| 176 |
-
save overall feedback, export the session, and push to HF.
|
| 177 |
-
"""
|
| 178 |
print(f"[DEBUG] export_results_with_data_processing called")
|
| 179 |
print(f"[DEBUG] overall_preference: '{overall_preference}'")
|
| 180 |
print(f"[DEBUG] comments: '{comments}'")
|
|
@@ -182,14 +215,21 @@ def create_app(data_manager, session_manager):
|
|
| 182 |
if not session:
|
| 183 |
return gr.update(), gr.update()
|
| 184 |
|
| 185 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
session_manager.process_mos_data(session, mos_data_json)
|
| 187 |
session_manager.process_ab_data(session, ab_data_json)
|
| 188 |
|
| 189 |
-
# Save overall feedback
|
| 190 |
feedback_response = {
|
| 191 |
"session_id": session["session_id"],
|
| 192 |
-
"overall_preference": overall_preference
|
| 193 |
"final_comments": comments or "",
|
| 194 |
"timestamp": time.time(),
|
| 195 |
}
|
|
@@ -200,13 +240,9 @@ def create_app(data_manager, session_manager):
|
|
| 200 |
export_data = session_manager.export_session(session["session_id"])
|
| 201 |
try:
|
| 202 |
push_session_to_hub(session["session_id"], export_data)
|
| 203 |
-
print(
|
| 204 |
-
f"[INFO] Successfully submitted session {session['session_id']} to Hub"
|
| 205 |
-
)
|
| 206 |
except Exception as e:
|
| 207 |
-
print(
|
| 208 |
-
f"[WARN] Failed to push session {session['session_id']} to Hub: {e}"
|
| 209 |
-
)
|
| 210 |
|
| 211 |
return (
|
| 212 |
gr.update(visible=False), # hide conclusion page
|
|
@@ -222,7 +258,7 @@ def create_app(data_manager, session_manager):
|
|
| 222 |
start_evaluation,
|
| 223 |
inputs=[],
|
| 224 |
outputs=[intro_page, sample_page, session_state],
|
| 225 |
-
js=
|
| 226 |
)
|
| 227 |
|
| 228 |
# Samples → MOS
|
|
@@ -230,7 +266,7 @@ def create_app(data_manager, session_manager):
|
|
| 230 |
show_mos_page,
|
| 231 |
inputs=[session_state],
|
| 232 |
outputs=[sample_page, mos_page, mos_dynamic_content],
|
| 233 |
-
js=
|
| 234 |
)
|
| 235 |
|
| 236 |
# MOS → AB Model (process MOS first)
|
|
@@ -238,52 +274,7 @@ def create_app(data_manager, session_manager):
|
|
| 238 |
fn=continue_from_mos_to_ab_model,
|
| 239 |
inputs=[session_state, mos_data_textbox],
|
| 240 |
outputs=[mos_page, ab_model_page, ab_model_dynamic_content],
|
| 241 |
-
js=
|
| 242 |
-
(session_data, current_mos_data) => {
|
| 243 |
-
window.scrollTo(0, 0);
|
| 244 |
-
|
| 245 |
-
// Collect MOS data from the current page
|
| 246 |
-
const ratings = {};
|
| 247 |
-
|
| 248 |
-
const selects = document.querySelectorAll('select[data-clip-id]');
|
| 249 |
-
const checkboxes = document.querySelectorAll('input[type="checkbox"][data-clip-id]');
|
| 250 |
-
const textareas = document.querySelectorAll('textarea[data-clip-id]');
|
| 251 |
-
|
| 252 |
-
console.log('[DEBUG] Found selects:', selects.length);
|
| 253 |
-
console.log('[DEBUG] Found checkboxes:', checkboxes.length);
|
| 254 |
-
console.log('[DEBUG] Found textareas:', textareas.length);
|
| 255 |
-
|
| 256 |
-
selects.forEach(select => {
|
| 257 |
-
const clipId = select.getAttribute('data-clip-id');
|
| 258 |
-
const dimension = select.getAttribute('data-dimension');
|
| 259 |
-
if (!ratings[clipId]) ratings[clipId] = {};
|
| 260 |
-
ratings[clipId][dimension] = select.value;
|
| 261 |
-
console.log('[DEBUG] Select:', clipId, dimension, select.value);
|
| 262 |
-
});
|
| 263 |
-
|
| 264 |
-
checkboxes.forEach(checkbox => {
|
| 265 |
-
const clipId = checkbox.getAttribute('data-clip-id');
|
| 266 |
-
const dimension = checkbox.getAttribute('data-dimension');
|
| 267 |
-
if (!ratings[clipId]) ratings[clipId] = {};
|
| 268 |
-
ratings[clipId][dimension] = checkbox.checked;
|
| 269 |
-
console.log('[DEBUG] Checkbox:', clipId, dimension, checkbox.checked);
|
| 270 |
-
});
|
| 271 |
-
|
| 272 |
-
textareas.forEach(textarea => {
|
| 273 |
-
const clipId = textarea.getAttribute('data-clip-id');
|
| 274 |
-
const dimension = textarea.getAttribute('data-dimension');
|
| 275 |
-
if (!ratings[clipId]) ratings[clipId] = {};
|
| 276 |
-
ratings[clipId][dimension] = textarea.value;
|
| 277 |
-
console.log('[DEBUG] Textarea:', clipId, dimension, textarea.value);
|
| 278 |
-
});
|
| 279 |
-
|
| 280 |
-
console.log('[DEBUG] Final ratings object:', ratings);
|
| 281 |
-
const ratingsJson = JSON.stringify(ratings);
|
| 282 |
-
console.log('[DEBUG] Ratings JSON:', ratingsJson);
|
| 283 |
-
|
| 284 |
-
return [session_data, ratingsJson];
|
| 285 |
-
}
|
| 286 |
-
"""
|
| 287 |
)
|
| 288 |
|
| 289 |
# AB Model → AB Gender (process AB model first)
|
|
@@ -291,58 +282,7 @@ def create_app(data_manager, session_manager):
|
|
| 291 |
continue_from_ab_model_to_ab_gender,
|
| 292 |
inputs=[session_state, ab_data_textbox],
|
| 293 |
outputs=[ab_model_page, ab_gender_page, ab_gender_dynamic_content],
|
| 294 |
-
js=
|
| 295 |
-
(session_data, current_ab_data) => {
|
| 296 |
-
window.scrollTo(0, 0);
|
| 297 |
-
|
| 298 |
-
// Collect A/B data from the current page
|
| 299 |
-
const ratings = {};
|
| 300 |
-
|
| 301 |
-
const radios = document.querySelectorAll('input[type="radio"]:checked');
|
| 302 |
-
const checkboxes = document.querySelectorAll('input[type="checkbox"][data-comparison-id]');
|
| 303 |
-
const textareas = document.querySelectorAll('textarea[data-comparison-id]');
|
| 304 |
-
|
| 305 |
-
console.log('[DEBUG] Found radios:', radios.length);
|
| 306 |
-
console.log('[DEBUG] Found checkboxes:', checkboxes.length);
|
| 307 |
-
console.log('[DEBUG] Found textareas:', textareas.length);
|
| 308 |
-
|
| 309 |
-
radios.forEach(radio => {
|
| 310 |
-
const compId = radio.getAttribute('data-comparison-id');
|
| 311 |
-
const clipAId = radio.getAttribute('data-clip-a-id');
|
| 312 |
-
const clipBId = radio.getAttribute('data-clip-b-id');
|
| 313 |
-
const compType = radio.getAttribute('data-type');
|
| 314 |
-
|
| 315 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 316 |
-
ratings[compId].clip_a_id = clipAId;
|
| 317 |
-
ratings[compId].clip_b_id = clipBId;
|
| 318 |
-
ratings[compId].comparison_type = compType;
|
| 319 |
-
ratings[compId].choice = radio.value;
|
| 320 |
-
console.log('[DEBUG] Radio:', compId, radio.value);
|
| 321 |
-
});
|
| 322 |
-
|
| 323 |
-
checkboxes.forEach(checkbox => {
|
| 324 |
-
const compId = checkbox.getAttribute('data-comparison-id');
|
| 325 |
-
const dimension = checkbox.getAttribute('data-dimension');
|
| 326 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 327 |
-
ratings[compId][dimension] = checkbox.checked;
|
| 328 |
-
console.log('[DEBUG] Checkbox:', compId, dimension, checkbox.checked);
|
| 329 |
-
});
|
| 330 |
-
|
| 331 |
-
textareas.forEach(textarea => {
|
| 332 |
-
const compId = textarea.getAttribute('data-comparison-id');
|
| 333 |
-
const dimension = textarea.getAttribute('data-dimension');
|
| 334 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 335 |
-
ratings[compId][dimension] = textarea.value;
|
| 336 |
-
console.log('[DEBUG] Textarea:', compId, dimension, textarea.value);
|
| 337 |
-
});
|
| 338 |
-
|
| 339 |
-
console.log('[DEBUG] Final A/B ratings object:', ratings);
|
| 340 |
-
const ratingsJson = JSON.stringify(ratings);
|
| 341 |
-
console.log('[DEBUG] A/B Ratings JSON:', ratingsJson);
|
| 342 |
-
|
| 343 |
-
return [session_data, ratingsJson];
|
| 344 |
-
}
|
| 345 |
-
"""
|
| 346 |
)
|
| 347 |
|
| 348 |
# AB Gender → Conclusion (process AB gender data)
|
|
@@ -350,58 +290,7 @@ def create_app(data_manager, session_manager):
|
|
| 350 |
complete_evaluation_with_ab_gender_data,
|
| 351 |
inputs=[session_state, ab_data_textbox],
|
| 352 |
outputs=[ab_gender_page, conclusion_page],
|
| 353 |
-
js=
|
| 354 |
-
(session_data, current_ab_data) => {
|
| 355 |
-
window.scrollTo(0, 0);
|
| 356 |
-
|
| 357 |
-
// Collect A/B gender data from the current page
|
| 358 |
-
const ratings = {};
|
| 359 |
-
|
| 360 |
-
const radios = document.querySelectorAll('input[type="radio"]:checked');
|
| 361 |
-
const checkboxes = document.querySelectorAll('input[type="checkbox"][data-comparison-id]');
|
| 362 |
-
const textareas = document.querySelectorAll('textarea[data-comparison-id]');
|
| 363 |
-
|
| 364 |
-
console.log('[DEBUG] Found radios:', radios.length);
|
| 365 |
-
console.log('[DEBUG] Found checkboxes:', checkboxes.length);
|
| 366 |
-
console.log('[DEBUG] Found textareas:', textareas.length);
|
| 367 |
-
|
| 368 |
-
radios.forEach(radio => {
|
| 369 |
-
const compId = radio.getAttribute('data-comparison-id');
|
| 370 |
-
const clipAId = radio.getAttribute('data-clip-a-id');
|
| 371 |
-
const clipBId = radio.getAttribute('data-clip-b-id');
|
| 372 |
-
const compType = radio.getAttribute('data-type');
|
| 373 |
-
|
| 374 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 375 |
-
ratings[compId].clip_a_id = clipAId;
|
| 376 |
-
ratings[compId].clip_b_id = clipBId;
|
| 377 |
-
ratings[compId].comparison_type = compType;
|
| 378 |
-
ratings[compId].choice = radio.value;
|
| 379 |
-
console.log('[DEBUG] Radio:', compId, radio.value);
|
| 380 |
-
});
|
| 381 |
-
|
| 382 |
-
checkboxes.forEach(checkbox => {
|
| 383 |
-
const compId = checkbox.getAttribute('data-comparison-id');
|
| 384 |
-
const dimension = checkbox.getAttribute('data-dimension');
|
| 385 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 386 |
-
ratings[compId][dimension] = checkbox.checked;
|
| 387 |
-
console.log('[DEBUG] Checkbox:', compId, dimension, checkbox.checked);
|
| 388 |
-
});
|
| 389 |
-
|
| 390 |
-
textareas.forEach(textarea => {
|
| 391 |
-
const compId = textarea.getAttribute('data-comparison-id');
|
| 392 |
-
const dimension = textarea.getAttribute('data-dimension');
|
| 393 |
-
if (!ratings[compId]) ratings[compId] = {};
|
| 394 |
-
ratings[compId][dimension] = textarea.value;
|
| 395 |
-
console.log('[DEBUG] Textarea:', compId, dimension, textarea.value);
|
| 396 |
-
});
|
| 397 |
-
|
| 398 |
-
console.log('[DEBUG] Final A/B gender ratings object:', ratings);
|
| 399 |
-
const ratingsJson = JSON.stringify(ratings);
|
| 400 |
-
console.log('[DEBUG] A/B Gender Ratings JSON:', ratingsJson);
|
| 401 |
-
|
| 402 |
-
return [session_data, ratingsJson];
|
| 403 |
-
}
|
| 404 |
-
"""
|
| 405 |
)
|
| 406 |
|
| 407 |
# Conclusion → Thank You (process any remaining data + export)
|
|
@@ -415,43 +304,7 @@ def create_app(data_manager, session_manager):
|
|
| 415 |
ab_data_textbox,
|
| 416 |
],
|
| 417 |
outputs=[conclusion_page, thank_you_page],
|
| 418 |
-
js=
|
| 419 |
-
(session_data, overall_feedback, final_comments, mos_data, ab_data) => {
|
| 420 |
-
window.scrollTo(0, 0);
|
| 421 |
-
|
| 422 |
-
console.log('[DEBUG] Export button clicked');
|
| 423 |
-
console.log('[DEBUG] overall_feedback from Gradio:', overall_feedback);
|
| 424 |
-
console.log('[DEBUG] final_comments from Gradio:', final_comments);
|
| 425 |
-
|
| 426 |
-
// Get actual values from the DOM elements
|
| 427 |
-
const radioInputs = document.querySelectorAll('input[type="radio"]');
|
| 428 |
-
let selectedRadio = null;
|
| 429 |
-
radioInputs.forEach(radio => {
|
| 430 |
-
if (radio.checked) {
|
| 431 |
-
selectedRadio = radio.value;
|
| 432 |
-
console.log('[DEBUG] Found checked radio:', selectedRadio);
|
| 433 |
-
}
|
| 434 |
-
});
|
| 435 |
-
|
| 436 |
-
const textareas = document.querySelectorAll('textarea');
|
| 437 |
-
let commentsValue = '';
|
| 438 |
-
textareas.forEach(textarea => {
|
| 439 |
-
// Skip hidden textboxes (MOS/AB data stores)
|
| 440 |
-
if (textarea.parentElement.style.display !== 'none' && !textarea.parentElement.classList.contains('hide')) {
|
| 441 |
-
if (textarea.value && textarea.value.trim() !== '') {
|
| 442 |
-
commentsValue = textarea.value;
|
| 443 |
-
console.log('[DEBUG] Found comments textarea:', commentsValue);
|
| 444 |
-
}
|
| 445 |
-
}
|
| 446 |
-
});
|
| 447 |
-
|
| 448 |
-
console.log('[DEBUG] Final selected radio:', selectedRadio);
|
| 449 |
-
console.log('[DEBUG] Final comments:', commentsValue);
|
| 450 |
-
|
| 451 |
-
// Return with actual values from DOM
|
| 452 |
-
return [session_data, selectedRadio, commentsValue, mos_data, ab_data];
|
| 453 |
-
}
|
| 454 |
-
"""
|
| 455 |
)
|
| 456 |
|
| 457 |
# Back navigation
|
|
@@ -459,35 +312,35 @@ def create_app(data_manager, session_manager):
|
|
| 459 |
lambda: (gr.update(visible=True), gr.update(visible=False)),
|
| 460 |
inputs=[],
|
| 461 |
outputs=[intro_page, sample_page],
|
| 462 |
-
js=
|
| 463 |
)
|
| 464 |
|
| 465 |
back_to_samples_btn.click(
|
| 466 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 467 |
inputs=[],
|
| 468 |
outputs=[mos_page, sample_page],
|
| 469 |
-
js=
|
| 470 |
)
|
| 471 |
|
| 472 |
back_to_mos_btn.click(
|
| 473 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 474 |
inputs=[],
|
| 475 |
outputs=[ab_model_page, mos_page],
|
| 476 |
-
js=
|
| 477 |
)
|
| 478 |
|
| 479 |
back_to_ab_model_btn.click(
|
| 480 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 481 |
inputs=[],
|
| 482 |
outputs=[ab_gender_page, ab_model_page],
|
| 483 |
-
js=
|
| 484 |
)
|
| 485 |
|
| 486 |
back_to_ab_gender_btn.click(
|
| 487 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 488 |
inputs=[],
|
| 489 |
outputs=[conclusion_page, ab_gender_page],
|
| 490 |
-
js=
|
| 491 |
)
|
| 492 |
|
| 493 |
return app
|
|
|
|
| 5 |
|
| 6 |
import time
|
| 7 |
import gradio as gr
|
|
|
|
| 8 |
import os
|
| 9 |
import sys
|
| 10 |
|
|
|
|
| 16 |
from backend.data_manager import DataManager
|
| 17 |
from backend.session_manager import SessionManager
|
| 18 |
from backend.hf_logging import push_session_to_hub
|
|
|
|
| 19 |
|
| 20 |
+
# Frontend pieces
|
| 21 |
from frontend.css import CSS
|
| 22 |
from frontend.pages.intro import build_intro_page
|
| 23 |
from frontend.pages.samples import build_sample_page
|
|
|
|
| 26 |
from frontend.pages.ab_gender import build_ab_gender_page, render_ab_gender_html
|
| 27 |
from frontend.pages.conclusion import build_conclusion_page
|
| 28 |
from frontend.pages.thank_you import build_thank_you_page
|
| 29 |
+
from frontend.utils.validation import (
|
| 30 |
+
validate_mos_completion,
|
| 31 |
+
validate_ab_completion,
|
| 32 |
+
validate_final_submission,
|
| 33 |
+
)
|
| 34 |
+
from frontend.utils.js_collectors import (
|
| 35 |
+
COLLECT_MOS_DATA_JS,
|
| 36 |
+
COLLECT_AB_DATA_JS,
|
| 37 |
+
COLLECT_CONCLUSION_DATA_JS,
|
| 38 |
+
SCROLL_TO_TOP_JS,
|
| 39 |
+
)
|
| 40 |
|
| 41 |
|
| 42 |
def create_app(data_manager, session_manager):
|
| 43 |
with gr.Blocks(title="ATC TTS Evaluation", css=CSS) as app:
|
| 44 |
# Global state
|
| 45 |
session_state = gr.State()
|
| 46 |
+
|
| 47 |
+
# Persistent data state - stores user inputs across page transitions
|
| 48 |
+
mos_data_state = gr.State(value="{}")
|
| 49 |
+
ab_data_state = gr.State(value="{}")
|
| 50 |
|
| 51 |
# Hidden textboxes used by the JS collectors on MOS / AB pages
|
| 52 |
mos_data_textbox = gr.Textbox(
|
|
|
|
| 138 |
)
|
| 139 |
|
| 140 |
def continue_from_mos_to_ab_model(session, mos_data_json):
|
| 141 |
+
"""Process MOS data and move to Model-vs-Model page."""
|
|
|
|
|
|
|
| 142 |
print(f"[DEBUG] continue_from_mos_to_ab_model called")
|
| 143 |
print(f"[DEBUG] MOS data JSON received: '{mos_data_json}'")
|
| 144 |
+
|
| 145 |
+
# Validate MOS completion
|
| 146 |
+
expected_clips = len(session.get("mos_clips", []))
|
| 147 |
+
is_valid, error_msg = validate_mos_completion(mos_data_json, expected_clips)
|
| 148 |
+
if not is_valid:
|
| 149 |
+
gr.Warning(error_msg)
|
| 150 |
+
return gr.update(), gr.update(), ""
|
| 151 |
+
|
| 152 |
+
# Process and navigate
|
| 153 |
session_manager.process_mos_data(session, mos_data_json)
|
| 154 |
ab_model_html = render_ab_model_html(session)
|
| 155 |
return (
|
|
|
|
| 159 |
)
|
| 160 |
|
| 161 |
def continue_from_ab_model_to_ab_gender(session, ab_data_json):
|
| 162 |
+
"""Process A/B model comparison data and move to Gender A/B page."""
|
|
|
|
|
|
|
| 163 |
print(f"[DEBUG] continue_from_ab_model_to_ab_gender called")
|
| 164 |
print(f"[DEBUG] AB data JSON received: '{ab_data_json}'")
|
| 165 |
+
|
| 166 |
+
# Validate AB model completion
|
| 167 |
+
expected_comparisons = len(session.get("ab_model_pairs", []))
|
| 168 |
+
is_valid, error_msg = validate_ab_completion(ab_data_json, expected_comparisons, "model comparisons")
|
| 169 |
+
if not is_valid:
|
| 170 |
+
gr.Warning(error_msg)
|
| 171 |
+
return gr.update(), gr.update(), ""
|
| 172 |
+
|
| 173 |
+
# Process and navigate
|
| 174 |
session_manager.process_ab_data(session, ab_data_json)
|
| 175 |
ab_gender_html = render_ab_gender_html(session)
|
| 176 |
return (
|
|
|
|
| 180 |
)
|
| 181 |
|
| 182 |
def complete_evaluation_with_ab_gender_data(session, ab_data_json):
|
| 183 |
+
"""Process A/B gender comparison data and move to conclusion page."""
|
| 184 |
+
print(f"[DEBUG] complete_evaluation_with_ab_gender_data called")
|
| 185 |
+
print(f"[DEBUG] AB gender data JSON received: '{ab_data_json}'")
|
| 186 |
+
|
| 187 |
+
# Validate AB gender completion
|
| 188 |
+
expected_comparisons = len(session.get("ab_gender_pairs", []))
|
| 189 |
+
is_valid, error_msg = validate_ab_completion(ab_data_json, expected_comparisons, "gender comparisons")
|
| 190 |
+
if not is_valid:
|
| 191 |
+
gr.Warning(error_msg)
|
| 192 |
+
return gr.update(), gr.update()
|
| 193 |
+
|
| 194 |
+
# Process and navigate
|
| 195 |
session_manager.process_ab_data(session, ab_data_json)
|
| 196 |
if session:
|
| 197 |
session["completed"] = True
|
|
|
|
| 207 |
mos_data_json,
|
| 208 |
ab_data_json,
|
| 209 |
):
|
| 210 |
+
"""Final submit: process data, validate overall preference, save, and push to HF."""
|
|
|
|
|
|
|
|
|
|
| 211 |
print(f"[DEBUG] export_results_with_data_processing called")
|
| 212 |
print(f"[DEBUG] overall_preference: '{overall_preference}'")
|
| 213 |
print(f"[DEBUG] comments: '{comments}'")
|
|
|
|
| 215 |
if not session:
|
| 216 |
return gr.update(), gr.update()
|
| 217 |
|
| 218 |
+
# Validate that overall preference is selected
|
| 219 |
+
is_valid, error_msg = validate_final_submission(overall_preference)
|
| 220 |
+
if not is_valid:
|
| 221 |
+
print(f"[ERROR] {error_msg}")
|
| 222 |
+
gr.Warning(error_msg)
|
| 223 |
+
return gr.update(), gr.update()
|
| 224 |
+
|
| 225 |
+
# Process all MOS / AB ratings
|
| 226 |
session_manager.process_mos_data(session, mos_data_json)
|
| 227 |
session_manager.process_ab_data(session, ab_data_json)
|
| 228 |
|
| 229 |
+
# Save overall feedback
|
| 230 |
feedback_response = {
|
| 231 |
"session_id": session["session_id"],
|
| 232 |
+
"overall_preference": overall_preference,
|
| 233 |
"final_comments": comments or "",
|
| 234 |
"timestamp": time.time(),
|
| 235 |
}
|
|
|
|
| 240 |
export_data = session_manager.export_session(session["session_id"])
|
| 241 |
try:
|
| 242 |
push_session_to_hub(session["session_id"], export_data)
|
| 243 |
+
print(f"[INFO] Successfully submitted session {session['session_id']} to Hub")
|
|
|
|
|
|
|
| 244 |
except Exception as e:
|
| 245 |
+
print(f"[WARN] Failed to push session {session['session_id']} to Hub: {e}")
|
|
|
|
|
|
|
| 246 |
|
| 247 |
return (
|
| 248 |
gr.update(visible=False), # hide conclusion page
|
|
|
|
| 258 |
start_evaluation,
|
| 259 |
inputs=[],
|
| 260 |
outputs=[intro_page, sample_page, session_state],
|
| 261 |
+
js=SCROLL_TO_TOP_JS
|
| 262 |
)
|
| 263 |
|
| 264 |
# Samples → MOS
|
|
|
|
| 266 |
show_mos_page,
|
| 267 |
inputs=[session_state],
|
| 268 |
outputs=[sample_page, mos_page, mos_dynamic_content],
|
| 269 |
+
js=SCROLL_TO_TOP_JS
|
| 270 |
)
|
| 271 |
|
| 272 |
# MOS → AB Model (process MOS first)
|
|
|
|
| 274 |
fn=continue_from_mos_to_ab_model,
|
| 275 |
inputs=[session_state, mos_data_textbox],
|
| 276 |
outputs=[mos_page, ab_model_page, ab_model_dynamic_content],
|
| 277 |
+
js=COLLECT_MOS_DATA_JS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
)
|
| 279 |
|
| 280 |
# AB Model → AB Gender (process AB model first)
|
|
|
|
| 282 |
continue_from_ab_model_to_ab_gender,
|
| 283 |
inputs=[session_state, ab_data_textbox],
|
| 284 |
outputs=[ab_model_page, ab_gender_page, ab_gender_dynamic_content],
|
| 285 |
+
js=COLLECT_AB_DATA_JS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
)
|
| 287 |
|
| 288 |
# AB Gender → Conclusion (process AB gender data)
|
|
|
|
| 290 |
complete_evaluation_with_ab_gender_data,
|
| 291 |
inputs=[session_state, ab_data_textbox],
|
| 292 |
outputs=[ab_gender_page, conclusion_page],
|
| 293 |
+
js=COLLECT_AB_DATA_JS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
)
|
| 295 |
|
| 296 |
# Conclusion → Thank You (process any remaining data + export)
|
|
|
|
| 304 |
ab_data_textbox,
|
| 305 |
],
|
| 306 |
outputs=[conclusion_page, thank_you_page],
|
| 307 |
+
js=COLLECT_CONCLUSION_DATA_JS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
)
|
| 309 |
|
| 310 |
# Back navigation
|
|
|
|
| 312 |
lambda: (gr.update(visible=True), gr.update(visible=False)),
|
| 313 |
inputs=[],
|
| 314 |
outputs=[intro_page, sample_page],
|
| 315 |
+
js=SCROLL_TO_TOP_JS
|
| 316 |
)
|
| 317 |
|
| 318 |
back_to_samples_btn.click(
|
| 319 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 320 |
inputs=[],
|
| 321 |
outputs=[mos_page, sample_page],
|
| 322 |
+
js=SCROLL_TO_TOP_JS
|
| 323 |
)
|
| 324 |
|
| 325 |
back_to_mos_btn.click(
|
| 326 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 327 |
inputs=[],
|
| 328 |
outputs=[ab_model_page, mos_page],
|
| 329 |
+
js=SCROLL_TO_TOP_JS
|
| 330 |
)
|
| 331 |
|
| 332 |
back_to_ab_model_btn.click(
|
| 333 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 334 |
inputs=[],
|
| 335 |
outputs=[ab_gender_page, ab_model_page],
|
| 336 |
+
js=SCROLL_TO_TOP_JS
|
| 337 |
)
|
| 338 |
|
| 339 |
back_to_ab_gender_btn.click(
|
| 340 |
lambda: (gr.update(visible=False), gr.update(visible=True)),
|
| 341 |
inputs=[],
|
| 342 |
outputs=[conclusion_page, ab_gender_page],
|
| 343 |
+
js=SCROLL_TO_TOP_JS
|
| 344 |
)
|
| 345 |
|
| 346 |
return app
|
frontend/pages/__pycache__/ab_gender.cpython-311.pyc
CHANGED
|
Binary files a/frontend/pages/__pycache__/ab_gender.cpython-311.pyc and b/frontend/pages/__pycache__/ab_gender.cpython-311.pyc differ
|
|
|
frontend/pages/__pycache__/ab_model.cpython-311.pyc
CHANGED
|
Binary files a/frontend/pages/__pycache__/ab_model.cpython-311.pyc and b/frontend/pages/__pycache__/ab_model.cpython-311.pyc differ
|
|
|
frontend/pages/__pycache__/conclusion.cpython-311.pyc
CHANGED
|
Binary files a/frontend/pages/__pycache__/conclusion.cpython-311.pyc and b/frontend/pages/__pycache__/conclusion.cpython-311.pyc differ
|
|
|
frontend/pages/__pycache__/mos.cpython-311.pyc
CHANGED
|
Binary files a/frontend/pages/__pycache__/mos.cpython-311.pyc and b/frontend/pages/__pycache__/mos.cpython-311.pyc differ
|
|
|
frontend/pages/ab_gender.py
CHANGED
|
@@ -34,6 +34,16 @@ def build_ab_gender_page():
|
|
| 34 |
value="<p>Loading gender comparisons...</p>", visible=True
|
| 35 |
)
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
with gr.Row():
|
| 38 |
back_btn = gr.Button(
|
| 39 |
"Back to Model Comparisons", variant="secondary"
|
|
|
|
| 34 |
value="<p>Loading gender comparisons...</p>", visible=True
|
| 35 |
)
|
| 36 |
|
| 37 |
+
gr.Markdown(
|
| 38 |
+
"""
|
| 39 |
+
<div style="background: #7f1d1d; padding: 12px; border-radius: 8px; border: 2px solid #ef4444; margin: 15px 0;">
|
| 40 |
+
<p style="margin: 0; color: #fca5a5; font-weight: bold;">
|
| 41 |
+
⚠️ Warning: Using the "Back" button will lose all your comparisons on this page!
|
| 42 |
+
</p>
|
| 43 |
+
</div>
|
| 44 |
+
"""
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
with gr.Row():
|
| 48 |
back_btn = gr.Button(
|
| 49 |
"Back to Model Comparisons", variant="secondary"
|
frontend/pages/ab_model.py
CHANGED
|
@@ -34,6 +34,16 @@ def build_ab_model_page():
|
|
| 34 |
value="<p>Loading model comparisons...</p>", visible=True
|
| 35 |
)
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
with gr.Row():
|
| 38 |
back_btn = gr.Button(
|
| 39 |
"Back to Model Ratings", variant="secondary"
|
|
|
|
| 34 |
value="<p>Loading model comparisons...</p>", visible=True
|
| 35 |
)
|
| 36 |
|
| 37 |
+
gr.Markdown(
|
| 38 |
+
"""
|
| 39 |
+
<div style="background: #7f1d1d; padding: 12px; border-radius: 8px; border: 2px solid #ef4444; margin: 15px 0;">
|
| 40 |
+
<p style="margin: 0; color: #fca5a5; font-weight: bold;">
|
| 41 |
+
⚠️ Warning: Using the "Back" button will lose all your comparisons on this page!
|
| 42 |
+
</p>
|
| 43 |
+
</div>
|
| 44 |
+
"""
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
with gr.Row():
|
| 48 |
back_btn = gr.Button(
|
| 49 |
"Back to Model Ratings", variant="secondary"
|
frontend/pages/conclusion.py
CHANGED
|
@@ -36,6 +36,16 @@ def build_conclusion_page():
|
|
| 36 |
lines=4,
|
| 37 |
)
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
with gr.Row():
|
| 40 |
back_btn = gr.Button(
|
| 41 |
"Back to Gender Comparisons", variant="secondary"
|
|
|
|
| 36 |
lines=4,
|
| 37 |
)
|
| 38 |
|
| 39 |
+
gr.Markdown(
|
| 40 |
+
"""
|
| 41 |
+
<div style="background: #7f1d1d; padding: 12px; border-radius: 8px; border: 2px solid #ef4444; margin: 15px 0;">
|
| 42 |
+
<p style="margin: 0; color: #fca5a5; font-weight: bold;">
|
| 43 |
+
⚠️ Warning: Using the "Back" button will lose all your previous submissions!
|
| 44 |
+
</p>
|
| 45 |
+
</div>
|
| 46 |
+
"""
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
with gr.Row():
|
| 50 |
back_btn = gr.Button(
|
| 51 |
"Back to Gender Comparisons", variant="secondary"
|
frontend/pages/mos.py
CHANGED
|
@@ -47,6 +47,16 @@ def build_mos_page():
|
|
| 47 |
|
| 48 |
mos_dynamic_content = gr.HTML(value="<p>Loading clips...</p>", visible=True)
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
with gr.Row():
|
| 51 |
back_btn = gr.Button("Back to Reference Audio", variant="secondary")
|
| 52 |
continue_btn = gr.Button(
|
|
|
|
| 47 |
|
| 48 |
mos_dynamic_content = gr.HTML(value="<p>Loading clips...</p>", visible=True)
|
| 49 |
|
| 50 |
+
gr.Markdown(
|
| 51 |
+
"""
|
| 52 |
+
<div style="background: #7f1d1d; padding: 12px; border-radius: 8px; border: 2px solid #ef4444; margin: 15px 0;">
|
| 53 |
+
<p style="margin: 0; color: #fca5a5; font-weight: bold;">
|
| 54 |
+
⚠️ Warning: Using the "Back" button will lose all your ratings on this page!
|
| 55 |
+
</p>
|
| 56 |
+
</div>
|
| 57 |
+
"""
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
with gr.Row():
|
| 61 |
back_btn = gr.Button("Back to Reference Audio", variant="secondary")
|
| 62 |
continue_btn = gr.Button(
|
frontend/utils/__pycache__/data_persistence.cpython-311.pyc
ADDED
|
Binary file (8.04 kB). View file
|
|
|
frontend/utils/__pycache__/js_collectors.cpython-311.pyc
ADDED
|
Binary file (5.91 kB). View file
|
|
|
frontend/utils/__pycache__/validation.cpython-311.pyc
ADDED
|
Binary file (5.31 kB). View file
|
|
|
frontend/utils/js_collectors.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# frontend/js_collectors.py
|
| 2 |
+
"""
|
| 3 |
+
JavaScript code snippets for collecting form data on the client side.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
# JavaScript to collect MOS ratings from the page
|
| 7 |
+
COLLECT_MOS_DATA_JS = """
|
| 8 |
+
(session_data, current_mos_data) => {
|
| 9 |
+
window.scrollTo(0, 0);
|
| 10 |
+
|
| 11 |
+
// Collect MOS data from the current page
|
| 12 |
+
const ratings = {};
|
| 13 |
+
|
| 14 |
+
const selects = document.querySelectorAll('select[data-clip-id]');
|
| 15 |
+
const checkboxes = document.querySelectorAll('input[type="checkbox"][data-clip-id]');
|
| 16 |
+
const textareas = document.querySelectorAll('textarea[data-clip-id]');
|
| 17 |
+
|
| 18 |
+
console.log('[DEBUG] Found selects:', selects.length);
|
| 19 |
+
console.log('[DEBUG] Found checkboxes:', checkboxes.length);
|
| 20 |
+
console.log('[DEBUG] Found textareas:', textareas.length);
|
| 21 |
+
|
| 22 |
+
selects.forEach(select => {
|
| 23 |
+
const clipId = select.getAttribute('data-clip-id');
|
| 24 |
+
const dimension = select.getAttribute('data-dimension');
|
| 25 |
+
if (!ratings[clipId]) ratings[clipId] = {};
|
| 26 |
+
ratings[clipId][dimension] = select.value;
|
| 27 |
+
console.log('[DEBUG] Select:', clipId, dimension, select.value);
|
| 28 |
+
});
|
| 29 |
+
|
| 30 |
+
checkboxes.forEach(checkbox => {
|
| 31 |
+
const clipId = checkbox.getAttribute('data-clip-id');
|
| 32 |
+
const dimension = checkbox.getAttribute('data-dimension');
|
| 33 |
+
if (!ratings[clipId]) ratings[clipId] = {};
|
| 34 |
+
ratings[clipId][dimension] = checkbox.checked;
|
| 35 |
+
console.log('[DEBUG] Checkbox:', clipId, dimension, checkbox.checked);
|
| 36 |
+
});
|
| 37 |
+
|
| 38 |
+
textareas.forEach(textarea => {
|
| 39 |
+
const clipId = textarea.getAttribute('data-clip-id');
|
| 40 |
+
const dimension = textarea.getAttribute('data-dimension');
|
| 41 |
+
if (!ratings[clipId]) ratings[clipId] = {};
|
| 42 |
+
ratings[clipId][dimension] = textarea.value;
|
| 43 |
+
console.log('[DEBUG] Textarea:', clipId, dimension, textarea.value);
|
| 44 |
+
});
|
| 45 |
+
|
| 46 |
+
console.log('[DEBUG] Final ratings object:', ratings);
|
| 47 |
+
const ratingsJson = JSON.stringify(ratings);
|
| 48 |
+
console.log('[DEBUG] Ratings JSON:', ratingsJson);
|
| 49 |
+
|
| 50 |
+
return [session_data, ratingsJson];
|
| 51 |
+
}
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
# JavaScript to collect A/B comparison data from the page
|
| 55 |
+
COLLECT_AB_DATA_JS = """
|
| 56 |
+
(session_data, current_ab_data) => {
|
| 57 |
+
window.scrollTo(0, 0);
|
| 58 |
+
|
| 59 |
+
// Collect A/B data from the current page
|
| 60 |
+
const ratings = {};
|
| 61 |
+
|
| 62 |
+
const radios = document.querySelectorAll('input[type="radio"]:checked');
|
| 63 |
+
const checkboxes = document.querySelectorAll('input[type="checkbox"][data-comparison-id]');
|
| 64 |
+
const textareas = document.querySelectorAll('textarea[data-comparison-id]');
|
| 65 |
+
|
| 66 |
+
console.log('[DEBUG] Found radios:', radios.length);
|
| 67 |
+
console.log('[DEBUG] Found checkboxes:', checkboxes.length);
|
| 68 |
+
console.log('[DEBUG] Found textareas:', textareas.length);
|
| 69 |
+
|
| 70 |
+
radios.forEach(radio => {
|
| 71 |
+
const compId = radio.getAttribute('data-comparison-id');
|
| 72 |
+
const clipAId = radio.getAttribute('data-clip-a-id');
|
| 73 |
+
const clipBId = radio.getAttribute('data-clip-b-id');
|
| 74 |
+
const compType = radio.getAttribute('data-type');
|
| 75 |
+
|
| 76 |
+
if (!ratings[compId]) ratings[compId] = {};
|
| 77 |
+
ratings[compId].clip_a_id = clipAId;
|
| 78 |
+
ratings[compId].clip_b_id = clipBId;
|
| 79 |
+
ratings[compId].comparison_type = compType;
|
| 80 |
+
ratings[compId].choice = radio.value;
|
| 81 |
+
console.log('[DEBUG] Radio:', compId, radio.value);
|
| 82 |
+
});
|
| 83 |
+
|
| 84 |
+
checkboxes.forEach(checkbox => {
|
| 85 |
+
const compId = checkbox.getAttribute('data-comparison-id');
|
| 86 |
+
const dimension = checkbox.getAttribute('data-dimension');
|
| 87 |
+
if (!ratings[compId]) ratings[compId] = {};
|
| 88 |
+
ratings[compId][dimension] = checkbox.checked;
|
| 89 |
+
console.log('[DEBUG] Checkbox:', compId, dimension, checkbox.checked);
|
| 90 |
+
});
|
| 91 |
+
|
| 92 |
+
textareas.forEach(textarea => {
|
| 93 |
+
const compId = textarea.getAttribute('data-comparison-id');
|
| 94 |
+
const dimension = textarea.getAttribute('data-dimension');
|
| 95 |
+
if (!ratings[compId]) ratings[compId] = {};
|
| 96 |
+
ratings[compId][dimension] = textarea.value;
|
| 97 |
+
console.log('[DEBUG] Textarea:', compId, dimension, textarea.value);
|
| 98 |
+
});
|
| 99 |
+
|
| 100 |
+
console.log('[DEBUG] Final A/B ratings object:', ratings);
|
| 101 |
+
const ratingsJson = JSON.stringify(ratings);
|
| 102 |
+
console.log('[DEBUG] A/B Ratings JSON:', ratingsJson);
|
| 103 |
+
|
| 104 |
+
return [session_data, ratingsJson];
|
| 105 |
+
}
|
| 106 |
+
"""
|
| 107 |
+
|
| 108 |
+
# JavaScript to collect overall feedback from conclusion page
|
| 109 |
+
COLLECT_CONCLUSION_DATA_JS = """
|
| 110 |
+
(session_data, overall_feedback, final_comments, mos_data, ab_data) => {
|
| 111 |
+
window.scrollTo(0, 0);
|
| 112 |
+
|
| 113 |
+
console.log('[DEBUG] Export button clicked');
|
| 114 |
+
console.log('[DEBUG] overall_feedback from Gradio:', overall_feedback);
|
| 115 |
+
console.log('[DEBUG] final_comments from Gradio:', final_comments);
|
| 116 |
+
|
| 117 |
+
// Get actual values from the DOM elements
|
| 118 |
+
const radioInputs = document.querySelectorAll('input[type="radio"]');
|
| 119 |
+
let selectedRadio = null;
|
| 120 |
+
radioInputs.forEach(radio => {
|
| 121 |
+
if (radio.checked) {
|
| 122 |
+
selectedRadio = radio.value;
|
| 123 |
+
console.log('[DEBUG] Found checked radio:', selectedRadio);
|
| 124 |
+
}
|
| 125 |
+
});
|
| 126 |
+
|
| 127 |
+
const textareas = document.querySelectorAll('textarea');
|
| 128 |
+
let commentsValue = '';
|
| 129 |
+
textareas.forEach(textarea => {
|
| 130 |
+
// Skip hidden textboxes (MOS/AB data stores)
|
| 131 |
+
if (textarea.parentElement.style.display !== 'none' && !textarea.parentElement.classList.contains('hide')) {
|
| 132 |
+
if (textarea.value && textarea.value.trim() !== '') {
|
| 133 |
+
commentsValue = textarea.value;
|
| 134 |
+
console.log('[DEBUG] Found comments textarea:', commentsValue);
|
| 135 |
+
}
|
| 136 |
+
}
|
| 137 |
+
});
|
| 138 |
+
|
| 139 |
+
console.log('[DEBUG] Final selected radio:', selectedRadio);
|
| 140 |
+
console.log('[DEBUG] Final comments:', commentsValue);
|
| 141 |
+
|
| 142 |
+
// Return with actual values from DOM
|
| 143 |
+
return [session_data, selectedRadio, commentsValue, mos_data, ab_data];
|
| 144 |
+
}
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
# Simple scroll to top JavaScript
|
| 148 |
+
SCROLL_TO_TOP_JS = "() => { window.scrollTo(0, 0); }"
|
frontend/utils/validation.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# frontend/validation.py
|
| 2 |
+
"""
|
| 3 |
+
Validation logic for MOS evaluation form data.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
+
import gradio as gr
|
| 8 |
+
from typing import Dict, Any, Tuple
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def validate_mos_completion(mos_data_json: str, expected_clips: int) -> Tuple[bool, str]:
|
| 12 |
+
"""
|
| 13 |
+
Validate that all MOS clips have been rated.
|
| 14 |
+
|
| 15 |
+
Returns:
|
| 16 |
+
(is_valid, error_message)
|
| 17 |
+
"""
|
| 18 |
+
if not mos_data_json or mos_data_json == "None":
|
| 19 |
+
return False, "⚠️ Please rate at least one dimension for each clip before continuing!"
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
ratings_data = json.loads(mos_data_json)
|
| 23 |
+
|
| 24 |
+
# Check that each clip is completed
|
| 25 |
+
# Clip is complete if: all 5 ratings are filled OR gender_mismatch is checked
|
| 26 |
+
clips_with_ratings = 0
|
| 27 |
+
for clip_id, ratings in ratings_data.items():
|
| 28 |
+
rating_dims = ["clarity", "pronunciation", "prosody", "naturalness", "overall"]
|
| 29 |
+
filled_ratings = [dim for dim in rating_dims if ratings.get(dim)]
|
| 30 |
+
|
| 31 |
+
all_ratings_filled = len(filled_ratings) == 5
|
| 32 |
+
has_gender_mismatch = ratings.get("gender_mismatch", False)
|
| 33 |
+
|
| 34 |
+
if all_ratings_filled or has_gender_mismatch:
|
| 35 |
+
clips_with_ratings += 1
|
| 36 |
+
|
| 37 |
+
if clips_with_ratings < expected_clips:
|
| 38 |
+
return False, f"⚠️ Please complete all clips before continuing! Each clip needs either all 5 ratings, or a gender flag. ({clips_with_ratings}/{expected_clips} completed)"
|
| 39 |
+
|
| 40 |
+
return True, ""
|
| 41 |
+
|
| 42 |
+
except json.JSONDecodeError:
|
| 43 |
+
return False, "⚠️ Error reading ratings. Please try again!"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def validate_ab_completion(ab_data_json: str, expected_comparisons: int, comparison_type: str = "comparisons") -> Tuple[bool, str]:
|
| 47 |
+
"""
|
| 48 |
+
Validate that all A/B comparisons have been completed.
|
| 49 |
+
|
| 50 |
+
A comparison is considered complete if either:
|
| 51 |
+
- A choice is selected (A, B, or tie), OR
|
| 52 |
+
- At least one gender mismatch checkbox is checked
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
ab_data_json: JSON string containing comparison data
|
| 56 |
+
expected_comparisons: Number of comparisons expected
|
| 57 |
+
comparison_type: Type of comparison for error message (e.g., "model comparisons", "gender comparisons")
|
| 58 |
+
|
| 59 |
+
Returns:
|
| 60 |
+
(is_valid, error_message)
|
| 61 |
+
"""
|
| 62 |
+
if not ab_data_json or ab_data_json == "None":
|
| 63 |
+
return False, f"⚠️ Please make a choice for each {comparison_type} before continuing!"
|
| 64 |
+
|
| 65 |
+
try:
|
| 66 |
+
comparisons_data = json.loads(ab_data_json)
|
| 67 |
+
|
| 68 |
+
# Check that each comparison is completed
|
| 69 |
+
# Complete if: has a choice OR has gender mismatch checked
|
| 70 |
+
completed_comparisons = 0
|
| 71 |
+
for comp_id, comparison in comparisons_data.items():
|
| 72 |
+
has_choice = bool(comparison.get("choice"))
|
| 73 |
+
has_gender_mismatch = (
|
| 74 |
+
comparison.get("gender_mismatch_a", False) or
|
| 75 |
+
comparison.get("gender_mismatch_b", False) or
|
| 76 |
+
comparison.get("gender_mismatch_male", False) or
|
| 77 |
+
comparison.get("gender_mismatch_female", False)
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
if has_choice or has_gender_mismatch:
|
| 81 |
+
completed_comparisons += 1
|
| 82 |
+
|
| 83 |
+
if completed_comparisons < expected_comparisons:
|
| 84 |
+
return False, f"⚠️ Please complete all {comparison_type} before continuing! Each needs either a choice or gender flag. ({completed_comparisons}/{expected_comparisons} completed)"
|
| 85 |
+
|
| 86 |
+
return True, ""
|
| 87 |
+
|
| 88 |
+
except json.JSONDecodeError:
|
| 89 |
+
return False, f"⚠️ Error reading {comparison_type}. Please try again!"
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def validate_overall_feedback(overall_preference: str) -> Tuple[bool, str]:
|
| 93 |
+
"""
|
| 94 |
+
Validate that overall feedback has been selected.
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
(is_valid, error_message)
|
| 98 |
+
"""
|
| 99 |
+
if not overall_preference or overall_preference == "None" or overall_preference == "":
|
| 100 |
+
return False, "⚠️ Please select an overall preference before submitting!"
|
| 101 |
+
|
| 102 |
+
return True, ""
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def validate_final_submission(overall_preference: str) -> Tuple[bool, str]:
|
| 106 |
+
"""
|
| 107 |
+
Validate final submission - only checks that overall preference is selected.
|
| 108 |
+
|
| 109 |
+
Returns:
|
| 110 |
+
(is_valid, error_message)
|
| 111 |
+
"""
|
| 112 |
+
# Only check that overall feedback is selected
|
| 113 |
+
return validate_overall_feedback(overall_preference)
|